fix: make --resume in batch pipeline actually skip completed work
Previously --resume re-queried every gradient in every row, overwriting existing values — an interrupted run could not be resumed cheaply. - bicorder_query.py: add --resume flag; skip gradients whose cells already have values, and report how many were skipped - bicorder_batch.py: pass --resume through to query; skip fully-complete rows before invoking the query script; report partial rows - bicorder_batch.py: import row/config helpers from bicorder_query instead of calling undefined names (would have crashed on --resume)
This commit is contained in:
1 parent
c571bf1c01
commit
fb3bebcea0
2 files changed
+50
-5
No files matched your search
@@ -54,6 +54,16 @@ def get_protocol_by_row(csv_path, row_number):
|
||||
return None
|
||||
|
||||
|
||||
def get_row_values(csv_path, row_number):
|
||||
"""Get all existing values for a row (1-indexed) as a dict of column -> value."""
|
||||
with open(csv_path, 'r', encoding='utf-8') as f:
|
||||
reader = csv.DictReader(f)
|
||||
for i, row in enumerate(reader, start=1):
|
||||
if i == row_number:
|
||||
return row
|
||||
return None
|
||||
|
||||
|
||||
def generate_gradient_prompt(protocol_descriptor, protocol_description, gradient):
|
||||
"""Generate a prompt for a single gradient evaluation."""
|
||||
return f"""Analyze this protocol: "{protocol_descriptor}"
|
||||
@@ -153,6 +163,8 @@ Example usage:
|
||||
default='../bicorder.json',
|
||||
help='Path to bicorder.json (default: ../bicorder.json)')
|
||||
parser.add_argument('-m', '--model', help='LLM model to use')
|
||||
parser.add_argument('--resume', action='store_true',
|
||||
help='Skip gradients that already have values in the CSV')
|
||||
parser.add_argument('--dry-run', action='store_true',
|
||||
help='Show prompts without calling LLM or updating CSV')
|
||||
|
||||
@@ -177,6 +189,15 @@ Example usage:
|
||||
bicorder_data = load_bicorder_config(args.bicorder)
|
||||
gradients = extract_gradients(bicorder_data)
|
||||
|
||||
# Load existing values for this row (for resume mode)
|
||||
existing_values = get_row_values(args.csv_path, args.row_number) or {}
|
||||
|
||||
# Count existing values for reporting
|
||||
if args.resume:
|
||||
already_filled = sum(1 for g in gradients if existing_values.get(g['column_name'], '').strip())
|
||||
if already_filled:
|
||||
print(f"Resume: {already_filled}/{len(gradients)} gradients already have values")
|
||||
|
||||
if args.dry_run:
|
||||
print(f"DRY RUN: Row {args.row_number}, {len(gradients)} gradients")
|
||||
print(f"Protocol: {protocol['descriptor']}\n")
|
||||
@@ -188,6 +209,11 @@ Example usage:
|
||||
for i, gradient in enumerate(gradients, 1):
|
||||
gradient_short = gradient['column_name'].replace('_', ' ')
|
||||
|
||||
# In resume mode, skip gradients that already have a value
|
||||
if args.resume and existing_values.get(gradient['column_name'], '').strip():
|
||||
print(f"[{i}/{len(gradients)}] {gradient_short}: SKIP (already has value)")
|
||||
continue
|
||||
|
||||
if not args.dry_run:
|
||||
print(f"[{i}/{len(gradients)}] Querying: {gradient_short}...", flush=True)
|
||||
|
||||
|
||||
Reference in new issue
Block a user