ntnsndr/protocol-bicorder
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Flatten data/readings/ → data/

Browse Source 
Remove the intermediate readings/ subdirectory level — dataset naming
(synthetic_YYYYMMDD, manual_YYYYMMDD) already encodes what the data is.
Update all path references across scripts and docs accordingly.

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
This commit is contained in:
Nathan Schneider
2026-03-20 17:46:23 -06:00
parent 1a80219a25
commit 60e83783ec
533 changed files with 97 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files

BIN
analysis/scripts/__pycache__/bicorder_classifier.cpython-314.pyc
View File

Binary file not shown.

8
analysis/scripts/bicorder_batch.py
View File

@@ -86,16 +86,16 @@ def main():
epilog="""
Example usage:
# Process all protocols
python3 bicorder_batch.py data/readings/synthetic_20251116/protocols_edited.csv -o data/readings/synthetic_20251116/readings.csv
python3 bicorder_batch.py data/synthetic_20251116/protocols_edited.csv -o data/synthetic_20251116/readings.csv
# Process specific rows
python3 bicorder_batch.py data/readings/synthetic_20251116/protocols_edited.csv -o data/readings/synthetic_20251116/readings.csv --start 1 --end 5
python3 bicorder_batch.py data/synthetic_20251116/protocols_edited.csv -o data/synthetic_20251116/readings.csv --start 1 --end 5
# With specific model
python3 bicorder_batch.py data/readings/synthetic_20251116/protocols_edited.csv -o data/readings/synthetic_20251116/readings.csv -m mistral
python3 bicorder_batch.py data/synthetic_20251116/protocols_edited.csv -o data/synthetic_20251116/readings.csv -m mistral
# With metadata
python3 bicorder_batch.py data/readings/synthetic_20251116/protocols_edited.csv -o data/readings/synthetic_20251116/readings.csv -a "Your Name" -s "Your standpoint"
python3 bicorder_batch.py data/synthetic_20251116/protocols_edited.csv -o data/synthetic_20251116/readings.csv -a "Your Name" -s "Your standpoint"
"""
)

2
analysis/scripts/bicorder_classifier.py
View File

@@ -70,7 +70,7 @@ class BicorderClassifier:
2: "Institutional/Bureaucratic"
}
def __init__(self, diagnostic_csv='data/readings/synthetic_20251116/readings.csv',
def __init__(self, diagnostic_csv='data/synthetic_20251116/readings.csv',
model_path=None):
"""Initialize classifier with pre-computed model data."""
if model_path is None:

10
analysis/scripts/classify_readings.py
View File

@@ -7,10 +7,10 @@ filled with the neutral value (5), so shortform readings can still be
classified — though with lower confidence.
Usage:
python3 scripts/classify_readings.py data/readings/manual_20260320/readings.csv
python3 scripts/classify_readings.py data/readings/manual_20260320/readings.csv \\
--training data/readings/synthetic_20251116/readings.csv \\
--output data/readings/manual_20260320/analysis/classifications.csv
python3 scripts/classify_readings.py data/manual_20260320/readings.csv
python3 scripts/classify_readings.py data/manual_20260320/readings.csv \\
--training data/synthetic_20251116/readings.csv \\
--output data/manual_20260320/analysis/classifications.csv
"""
import argparse
@@ -29,7 +29,7 @@ def main():
parser.add_argument('input_csv', help='Readings CSV to classify')
parser.add_argument(
'--training',
default='data/readings/synthetic_20251116/readings.csv',
default='data/synthetic_20251116/readings.csv',
help='Training CSV for classifier (default: synthetic_20251116)'
)
parser.add_argument(

8
analysis/scripts/compare_analyses.py
View File

@@ -157,11 +157,11 @@ def compare_analyses(reference_file, comparison_files):
if __name__ == "__main__":
# Define file paths
reference_file = "data/readings/synthetic_20251116/readings_manual.csv"
reference_file = "data/synthetic_20251116/readings_manual.csv"
comparison_files = [
"data/readings/synthetic_20251116/readings_gemma3-12b.csv",
"data/readings/synthetic_20251116/readings_gpt-oss.csv",
"data/readings/synthetic_20251116/readings_mistral.csv"
"data/synthetic_20251116/readings_gemma3-12b.csv",
"data/synthetic_20251116/readings_gpt-oss.csv",
"data/synthetic_20251116/readings_mistral.csv"
]
# Check if files exist

6
analysis/scripts/convert_csv_to_json.py
View File

@@ -137,11 +137,11 @@ def main():
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Example usage:
python3 scripts/convert_csv_to_json.py data/readings/synthetic_20251116/readings.csv
python3 scripts/convert_csv_to_json.py data/readings/manual_20260101/readings.csv --output-dir data/readings/manual_20260101/json
python3 scripts/convert_csv_to_json.py data/synthetic_20251116/readings.csv
python3 scripts/convert_csv_to_json.py data/manual_20260101/readings.csv --output-dir data/manual_20260101/json
"""
)
parser.add_argument('input_csv', help='Diagnostic readings CSV (e.g. data/readings/synthetic_20251116/readings.csv)')
parser.add_argument('input_csv', help='Diagnostic readings CSV (e.g. data/synthetic_20251116/readings.csv)')
parser.add_argument('--output-dir', default=None,
help='Output directory for JSON files (default: <dataset_dir>/json)')
parser.add_argument('--bicorder', default='../bicorder.json',

10
analysis/scripts/export_model_for_js.py
View File

@@ -9,8 +9,8 @@ When gradients are renamed in bicorder.json, add the old→new mapping to
COLUMN_RENAMES so the training CSV columns are correctly aligned.
Usage:
python3 scripts/export_model_for_js.py data/readings/synthetic_20251116/readings.csv
python3 scripts/export_model_for_js.py data/readings/manual_20260101/readings.csv --output bicorder_model.json
python3 scripts/export_model_for_js.py data/synthetic_20251116/readings.csv
python3 scripts/export_model_for_js.py data/manual_20260101/readings.csv --output bicorder_model.json
"""
import argparse
@@ -60,11 +60,11 @@ def main():
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Example usage:
python3 scripts/export_model_for_js.py data/readings/synthetic_20251116/readings.csv
python3 scripts/export_model_for_js.py data/readings/manual_20260101/readings.csv --output bicorder_model.json
python3 scripts/export_model_for_js.py data/synthetic_20251116/readings.csv
python3 scripts/export_model_for_js.py data/manual_20260101/readings.csv --output bicorder_model.json
"""
)
parser.add_argument('input_csv', help='Diagnostic readings CSV (e.g. data/readings/synthetic_20251116/readings.csv)')
parser.add_argument('input_csv', help='Diagnostic readings CSV (e.g. data/synthetic_20251116/readings.csv)')
parser.add_argument('--output', default='bicorder_model.json',
help='Output model JSON path (default: bicorder_model.json)')
args = parser.parse_args()

12
analysis/scripts/json_to_csv.py
View File

@@ -10,9 +10,9 @@ Null gradient values (e.g., shortform readings that skip non-key dimensions) are
written as empty cells so downstream analysis can treat them as NaN.
Usage:
python3 scripts/json_to_csv.py data/readings/manual_20260320/json/
python3 scripts/json_to_csv.py data/readings/manual_20260320/json/ -o data/readings/manual_20260320/readings.csv
python3 scripts/json_to_csv.py data/readings/manual_20260320/json/ --shortform-only
python3 scripts/json_to_csv.py data/manual_20260320/json/
python3 scripts/json_to_csv.py data/manual_20260320/json/ -o data/manual_20260320/readings.csv
python3 scripts/json_to_csv.py data/manual_20260320/json/ --shortform-only
"""
import argparse
@@ -100,9 +100,9 @@ def main():
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Example usage:
python3 scripts/json_to_csv.py data/readings/manual_20260320/json/
python3 scripts/json_to_csv.py data/readings/manual_20260320/json/ --shortform-only
python3 scripts/json_to_csv.py data/readings/manual_20260320/json/ -o data/readings/manual_20260320/readings.csv
python3 scripts/json_to_csv.py data/manual_20260320/json/
python3 scripts/json_to_csv.py data/manual_20260320/json/ --shortform-only
python3 scripts/json_to_csv.py data/manual_20260320/json/ -o data/manual_20260320/readings.csv
"""
)
parser.add_argument('json_dir', help='Directory containing bicorder JSON reading files')

10
analysis/scripts/lda_visualization.py
View File

@@ -3,8 +3,8 @@
Create LDA visualization to maximize cluster separation.
Usage:
python3 scripts/lda_visualization.py data/readings/synthetic_20251116.csv
python3 scripts/lda_visualization.py data/readings/synthetic_20251116.csv --results-dir analysis_results/synthetic_20251116
python3 scripts/lda_visualization.py data/synthetic_20251116.csv
python3 scripts/lda_visualization.py data/synthetic_20251116.csv --results-dir analysis_results/synthetic_20251116
"""
import argparse
@@ -22,11 +22,11 @@ def main():
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Example usage:
python3 scripts/lda_visualization.py data/readings/synthetic_20251116/readings.csv
python3 scripts/lda_visualization.py data/readings/manual_20260101/readings.csv --analysis-dir data/readings/manual_20260101/analysis
python3 scripts/lda_visualization.py data/synthetic_20251116/readings.csv
python3 scripts/lda_visualization.py data/manual_20260101/readings.csv --analysis-dir data/manual_20260101/analysis
"""
)
parser.add_argument('input_csv', help='Diagnostic readings CSV (e.g. data/readings/synthetic_20251116/readings.csv)')
parser.add_argument('input_csv', help='Diagnostic readings CSV (e.g. data/synthetic_20251116/readings.csv)')
parser.add_argument('--analysis-dir', default=None,
help='Analysis directory (default: <dataset_dir>/analysis)')
args = parser.parse_args()

8
analysis/scripts/multivariate_analysis.py
View File

@@ -764,13 +764,13 @@ def main():
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Examples:
python3 scripts/multivariate_analysis.py data/readings/synthetic_20251116/readings.csv
python3 scripts/multivariate_analysis.py data/readings/synthetic_20251116/readings.csv --output data/readings/synthetic_20251116/analysis
python3 scripts/multivariate_analysis.py data/readings/synthetic_20251116/readings.csv --analyses clustering pca
python3 scripts/multivariate_analysis.py data/synthetic_20251116/readings.csv
python3 scripts/multivariate_analysis.py data/synthetic_20251116/readings.csv --output data/synthetic_20251116/analysis
python3 scripts/multivariate_analysis.py data/synthetic_20251116/readings.csv --analyses clustering pca
"""
)
parser.add_argument('csv_file', help='Diagnostic readings CSV (e.g. data/readings/synthetic_20251116/readings.csv)')
parser.add_argument('csv_file', help='Diagnostic readings CSV (e.g. data/synthetic_20251116/readings.csv)')
parser.add_argument('--output', '-o', default=None,
help='Output directory (default: <dataset_dir>/analysis)')
parser.add_argument('--min-coverage', type=float, default=0.0,

10
analysis/scripts/review_analysis.py
View File

@@ -3,8 +3,8 @@
Comprehensive review of the analysis for errors and inconsistencies.
Usage:
python3 scripts/review_analysis.py data/readings/synthetic_20251116.csv
python3 scripts/review_analysis.py data/readings/manual_20260101.csv --results-dir analysis_results/manual_20260101
python3 scripts/review_analysis.py data/synthetic_20251116.csv
python3 scripts/review_analysis.py data/manual_20260101.csv --results-dir analysis_results/manual_20260101
"""
import argparse
@@ -19,11 +19,11 @@ def main():
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Example usage:
python3 scripts/review_analysis.py data/readings/synthetic_20251116/readings.csv
python3 scripts/review_analysis.py data/readings/manual_20260101/readings.csv --analysis-dir data/readings/manual_20260101/analysis
python3 scripts/review_analysis.py data/synthetic_20251116/readings.csv
python3 scripts/review_analysis.py data/manual_20260101/readings.csv --analysis-dir data/manual_20260101/analysis
"""
)
parser.add_argument('input_csv', help='Diagnostic readings CSV (e.g. data/readings/synthetic_20251116/readings.csv)')
parser.add_argument('input_csv', help='Diagnostic readings CSV (e.g. data/synthetic_20251116/readings.csv)')
parser.add_argument('--analysis-dir', default=None,
help='Analysis directory (default: <dataset_dir>/analysis)')
args = parser.parse_args()

10
analysis/scripts/sync_readings.sh
View File

@@ -4,10 +4,10 @@
# Reads remote URL and subdirectory from a .sync_source file in the dataset directory.
#
# Usage:
# scripts/sync_readings.sh data/readings/manual_20260320
# scripts/sync_readings.sh data/readings/manual_20260320 --no-analysis
# scripts/sync_readings.sh data/readings/manual_20260320 --min-coverage 0.8
# scripts/sync_readings.sh data/readings/manual_20260320 --training data/readings/synthetic_20251116/readings.csv
# scripts/sync_readings.sh data/manual_20260320
# scripts/sync_readings.sh data/manual_20260320 --no-analysis
# scripts/sync_readings.sh data/manual_20260320 --min-coverage 0.8
# scripts/sync_readings.sh data/manual_20260320 --training data/synthetic_20251116/readings.csv
#
# .sync_source format:
# REMOTE_URL=https://git.example.org/user/repo
@@ -18,7 +18,7 @@ set -euo pipefail
DATASET_DIR="${1:?Usage: $0 <dataset_dir> [--no-analysis] [--min-coverage N]}"
RUN_ANALYSIS=true
MIN_COVERAGE=0.8
TRAINING_CSV="data/readings/synthetic_20251116/readings.csv"
TRAINING_CSV="data/synthetic_20251116/readings.csv"
shift || true
while [[ $# -gt 0 ]]; do

10
analysis/scripts/visualize_clusters.py
View File

@@ -3,8 +3,8 @@
Create visualizations of k-means clusters overlaid on dimensionality reduction plots.
Usage:
python3 scripts/visualize_clusters.py data/readings/synthetic_20251116.csv
python3 scripts/visualize_clusters.py data/readings/manual_20260101.csv --results-dir analysis_results/manual_20260101
python3 scripts/visualize_clusters.py data/synthetic_20251116.csv
python3 scripts/visualize_clusters.py data/manual_20260101.csv --results-dir analysis_results/manual_20260101
"""
import argparse
@@ -20,11 +20,11 @@ def main():
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog="""
Example usage:
python3 scripts/visualize_clusters.py data/readings/synthetic_20251116/readings.csv
python3 scripts/visualize_clusters.py data/readings/manual_20260101/readings.csv --analysis-dir data/readings/manual_20260101/analysis
python3 scripts/visualize_clusters.py data/synthetic_20251116/readings.csv
python3 scripts/visualize_clusters.py data/manual_20260101/readings.csv --analysis-dir data/manual_20260101/analysis
"""
)
parser.add_argument('input_csv', help='Diagnostic readings CSV (e.g. data/readings/synthetic_20251116/readings.csv)')
parser.add_argument('input_csv', help='Diagnostic readings CSV (e.g. data/synthetic_20251116/readings.csv)')
parser.add_argument('--analysis-dir', default=None,
help='Analysis directory (default: <dataset_dir>/analysis)')
args = parser.parse_args()