Protocolbot
6ae77a4f9b
Strategy: key runs by bicorder version (not date), promote shared inputs
to analysis/data/, and stamp every output with its bicorder_version so
a re-run on edited gradients is self-describing.
Data layout:
- Promote the shared protocol inputs out of the run directory:
analysis/data/protocols_edited.csv (411 cleaned protocols)
analysis/data/protocols_raw.csv (774 un-cleaned entries)
- Rename the v1.2.6 synthetic run:
data/synthetic_20251116/ -> data/synthetic_1.2.6/
so the bicorder version it was scored against is explicit (gradient
structure changes between versions make date-based names ambiguous)
Provenance:
- bicorder_analyze.py now writes a 'bicorder_version' column into every
output readings.csv, recording which gradient structure produced it
Scripts:
- Update the real code defaults that pointed at the old run path
(bicorder_classifier.py, classify_readings.py, sync_readings.sh,
compare_analyses.py) and refresh docstring/help examples
- Remove a stray committed __pycache__/.pyc
Docs: analysis/README.md documents the new layout + how to add a run;
WORKFLOW.md, TEST_COMMANDS.md, INTEGRATION_GUIDE.md paths updated.
108 lines
3.0 KiB
Bash
Executable File
108 lines
3.0 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Sync a readings dataset from a remote git repository, then regenerate CSV and analysis.
|
|
#
|
|
# Reads remote URL and subdirectory from a .sync_source file in the dataset directory.
|
|
#
|
|
# Usage:
|
|
# scripts/sync_readings.sh data/manual_20260320
|
|
# scripts/sync_readings.sh data/manual_20260320 --no-analysis
|
|
# scripts/sync_readings.sh data/manual_20260320 --min-coverage 0.8
|
|
# scripts/sync_readings.sh data/manual_20260320 --training data/synthetic_1.2.6/readings.csv
|
|
#
|
|
# .sync_source format:
|
|
# REMOTE_URL=https://git.example.org/user/repo
|
|
# REMOTE_SUBDIR=readings
|
|
|
|
set -euo pipefail
|
|
|
|
DATASET_DIR="${1:?Usage: $0 <dataset_dir> [--no-analysis] [--min-coverage N]}"
|
|
RUN_ANALYSIS=true
|
|
MIN_COVERAGE=0.8
|
|
TRAINING_CSV="data/synthetic_1.2.6/readings.csv"
|
|
|
|
shift || true
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--no-analysis) RUN_ANALYSIS=false ;;
|
|
--min-coverage) MIN_COVERAGE="$2"; shift ;;
|
|
--training) TRAINING_CSV="$2"; shift ;;
|
|
*) echo "Unknown option: $1" >&2; exit 1 ;;
|
|
esac
|
|
shift
|
|
done
|
|
|
|
SYNC_SOURCE="$DATASET_DIR/.sync_source"
|
|
if [[ ! -f "$SYNC_SOURCE" ]]; then
|
|
echo "Error: $SYNC_SOURCE not found. Create it with REMOTE_URL and REMOTE_SUBDIR." >&2
|
|
exit 1
|
|
fi
|
|
|
|
# Load config
|
|
REMOTE_URL=$(grep '^REMOTE_URL=' "$SYNC_SOURCE" | cut -d= -f2-)
|
|
REMOTE_SUBDIR=$(grep '^REMOTE_SUBDIR=' "$SYNC_SOURCE" | cut -d= -f2-)
|
|
|
|
if [[ -z "$REMOTE_URL" ]]; then
|
|
echo "Error: REMOTE_URL not set in $SYNC_SOURCE" >&2
|
|
exit 1
|
|
fi
|
|
|
|
REMOTE_SUBDIR="${REMOTE_SUBDIR:-readings}"
|
|
JSON_DIR="$DATASET_DIR/json"
|
|
|
|
echo "========================================"
|
|
echo "Syncing: $DATASET_DIR"
|
|
echo "From: $REMOTE_URL/$REMOTE_SUBDIR"
|
|
echo "========================================"
|
|
|
|
# Clone remote to temp dir and copy JSON files
|
|
TMPDIR=$(mktemp -d)
|
|
trap "rm -rf '$TMPDIR'" EXIT
|
|
|
|
echo ""
|
|
echo "Fetching remote data..."
|
|
git clone --depth 1 --quiet "$REMOTE_URL" "$TMPDIR"
|
|
|
|
SRC="$TMPDIR/$REMOTE_SUBDIR"
|
|
if [[ ! -d "$SRC" ]]; then
|
|
echo "Error: subdirectory '$REMOTE_SUBDIR' not found in remote repo." >&2
|
|
exit 1
|
|
fi
|
|
|
|
NEW=$(find "$SRC" -name '*.json' | wc -l | tr -d ' ')
|
|
mkdir -p "$JSON_DIR"
|
|
cp "$SRC"/*.json "$JSON_DIR"/
|
|
echo "Copied $NEW JSON files → $JSON_DIR"
|
|
|
|
# Determine VENV python
|
|
PYTHON=python3
|
|
if [[ -f ".venv/bin/python3" ]]; then
|
|
PYTHON=".venv/bin/python3"
|
|
fi
|
|
|
|
# Regenerate CSV
|
|
echo ""
|
|
echo "Regenerating readings.csv..."
|
|
"$PYTHON" scripts/json_to_csv.py "$JSON_DIR" -o "$DATASET_DIR/readings.csv"
|
|
|
|
if [[ "$RUN_ANALYSIS" == true ]]; then
|
|
echo ""
|
|
echo "Running multivariate analysis (--min-coverage $MIN_COVERAGE)..."
|
|
"$PYTHON" scripts/multivariate_analysis.py \
|
|
"$DATASET_DIR/readings.csv" \
|
|
--min-coverage "$MIN_COVERAGE" \
|
|
--analyses clustering pca correlation importance
|
|
|
|
echo ""
|
|
echo "Generating LDA visualization..."
|
|
"$PYTHON" scripts/lda_visualization.py "$DATASET_DIR/readings.csv"
|
|
|
|
echo ""
|
|
echo "Classifying readings (training: $TRAINING_CSV)..."
|
|
"$PYTHON" scripts/classify_readings.py \
|
|
"$DATASET_DIR/readings.csv" \
|
|
--training "$TRAINING_CSV"
|
|
fi
|
|
|
|
echo ""
|
|
echo "Done. Dataset: $DATASET_DIR"
|