From 1a80219a25f8318dd1473a9a87aa1bedbbde717d Mon Sep 17 00:00:00 2001 From: Nathan Schneider Date: Fri, 20 Mar 2026 17:39:25 -0600 Subject: [PATCH] Remove web/ prototype; update docs to reflect app integration The web/ directory (bicorder-classifier.js, .d.ts, test-classifier.mjs) was a prototype superseded by bicorder-app/src/bicorder-classifier.ts. The only integration point between this analysis directory and the app is bicorder_model.json, which Vite reads at build time. Co-Authored-By: Claude Sonnet 4.6 --- analysis/INTEGRATION_GUIDE.md | 42 +--- analysis/README.md | 6 +- analysis/WORKFLOW.md | 2 +- analysis/web/bicorder-classifier.d.ts | 98 -------- analysis/web/bicorder-classifier.js | 335 -------------------------- analysis/web/test-classifier.mjs | 41 ---- 6 files changed, 17 insertions(+), 507 deletions(-) delete mode 100644 analysis/web/bicorder-classifier.d.ts delete mode 100644 analysis/web/bicorder-classifier.js delete mode 100644 analysis/web/test-classifier.mjs diff --git a/analysis/INTEGRATION_GUIDE.md b/analysis/INTEGRATION_GUIDE.md index 473b91f..83b05cc 100644 --- a/analysis/INTEGRATION_GUIDE.md +++ b/analysis/INTEGRATION_GUIDE.md @@ -19,30 +19,18 @@ This ensures the web app and model stay in sync without complex backward compati ## Files -- `bicorder_model.json` - Trained model parameters (5KB, embed in app) -- `web/bicorder-classifier.js` - JavaScript implementation -- `web/bicorder-classifier.d.ts` - TypeScript type definitions +- `bicorder_model.json` - Trained model parameters (~5KB); read by `bicorder-app` at build time from `../analysis/bicorder_model.json` +- `bicorder-app/src/bicorder-classifier.ts` - TypeScript classifier implementation (lives in the app, not here) + +The model is the only artifact produced by this analysis directory that the app consumes. Regenerate it after re-running analysis on the synthetic dataset: + +```bash +python3 scripts/export_model_for_js.py data/readings/synthetic_20251116/readings.csv +``` ## Quick Start -### 1. Copy Model File - -Copy `bicorder_model.json` to your web app's public/static assets: - -```bash -cp bicorder_model.json ../path/to/bicorder/public/ -``` - -### 2. Install Classifier - -Copy the JavaScript module to your source directory: - -```bash -cp web/bicorder-classifier.js ../path/to/bicorder/src/lib/ -cp web/bicorder-classifier.d.ts ../path/to/bicorder/src/lib/ -``` - -### 3. Basic Usage +### Basic Usage ```javascript import { loadClassifier } from './lib/bicorder-classifier.js'; @@ -343,11 +331,11 @@ Check if enough key dimensions are provided. ## Testing -Test the classifier with example protocols: +Test the classifier with example protocols (run from within `bicorder-app`): ```javascript -import { BicorderClassifier } from './bicorder-classifier.js'; -import modelData from './bicorder_model.json'; +import { BicorderClassifier } from './bicorder-classifier'; +import modelData from '../../analysis/bicorder_model.json'; const classifier = new BicorderClassifier(modelData); @@ -395,8 +383,4 @@ console.log(classifier.predict(boundary)); ## Questions? -See the test in `web/test-classifier.mjs` for working examples, or test with: - -```bash -node web/test-classifier.mjs -``` +See `bicorder-app/src/bicorder-classifier.ts` for the live implementation, and `bicorder-app/src/App.svelte` for how it's wired into the form. diff --git a/analysis/README.md b/analysis/README.md index 96b481c..93e4e5c 100644 --- a/analysis/README.md +++ b/analysis/README.md @@ -437,8 +437,8 @@ This simple version-matching approach ensures compatibility without complex stru ### Files -- `bicorder_model.json` (5KB) - Trained LDA model with coefficients and scaler parameters -- `web/bicorder-classifier.js` - JavaScript implementation for real-time classification in web app +- `bicorder_model.json` (~5KB) - Trained LDA model with coefficients and scaler parameters; read by `bicorder-app` at build time +- `bicorder-app/src/bicorder-classifier.ts` - TypeScript classifier implementation in the web app - `ascii_bicorder.py` (updated) - Python script now calculates automated analysis values - `../bicorder.json` (updated) - Added bureaucratic ↔ relational gradient to analysis section @@ -450,7 +450,7 @@ The calculation happens automatically when generating bicorder output: python3 ascii_bicorder.py bicorder.json bicorder.txt ``` -For web integration, see `INTEGRATION_GUIDE.md` for details on using `web/bicorder-classifier.js` to provide real-time classification as users fill out diagnostics. +For web integration, see `INTEGRATION_GUIDE.md`. The app (`bicorder-app/`) has its own classifier implementation and reads `bicorder_model.json` from this directory at build time. ### Key Features diff --git a/analysis/WORKFLOW.md b/analysis/WORKFLOW.md index d29e3c5..60e4062 100644 --- a/analysis/WORKFLOW.md +++ b/analysis/WORKFLOW.md @@ -23,7 +23,7 @@ The scripts automatically draw the gradients from the current state of the [bico 7. **scripts/lda_visualization.py** - Generate LDA cluster separation plot and projection data 8. **scripts/classify_readings.py** - Apply the synthetic-trained LDA classifier to all readings; saves `analysis/classifications.csv` 9. **scripts/visualize_clusters.py** - Additional cluster visualizations -10. **scripts/export_model_for_js.py** - Export trained model to `bicorder_model.json` for the web classifier +10. **scripts/export_model_for_js.py** - Export trained model to `bicorder_model.json` (read by `bicorder-app` at build time) ## Syncing a manual readings dataset diff --git a/analysis/web/bicorder-classifier.d.ts b/analysis/web/bicorder-classifier.d.ts deleted file mode 100644 index d242356..0000000 --- a/analysis/web/bicorder-classifier.d.ts +++ /dev/null @@ -1,98 +0,0 @@ -/** - * Type definitions for Bicorder Cluster Classifier - */ - -export interface ModelData { - version: string; - generated: string; - dimensions: string[]; - key_dimensions: string[]; - cluster_names: { - '1': string; - '2': string; - }; - cluster_descriptions: { - '1': string; - '2': string; - }; - scaler: { - mean: number[]; - scale: number[]; - }; - lda: { - coefficients: number[]; - intercept: number; - }; - cluster_centroids_scaled: { - '1': number[]; - '2': number[]; - }; - cluster_means_original: { - '1': number[]; - '2': number[]; - }; - thresholds: { - confidence_low: number; - completeness_low: number; - boundary_distance_low: number; - }; - metadata: { - total_protocols: number; - cluster_1_count: number; - cluster_2_count: number; - }; -} - -export interface Ratings { - [dimensionName: string]: number | null | undefined; -} - -export interface PredictionResult { - cluster: 1 | 2; - clusterName: string; - confidence: number; - completeness: number; - recommendedForm: 'short' | 'long'; -} - -export interface DetailedPredictionResult extends PredictionResult { - ldaScore: number; - distanceToBoundary: number; - dimensionsProvided: number; - dimensionsTotal: number; - keyDimensionsProvided: number; - keyDimensionsTotal: number; - distancesToCentroids: { - '1': number; - '2': number; - }; - rawConfidence: number; -} - -export interface ShortFormAssessment { - ready: boolean; - keyDimensionsProvided: number; - keyDimensionsTotal: number; - coverage: number; - missingKeyDimensions: string[]; -} - -export interface PredictOptions { - detailed?: boolean; -} - -export class BicorderClassifier { - constructor(model: ModelData); - - predict(ratings: Ratings, options?: { detailed: false }): PredictionResult; - predict(ratings: Ratings, options: { detailed: true }): DetailedPredictionResult; - predict(ratings: Ratings, options?: PredictOptions): PredictionResult | DetailedPredictionResult; - - explainClassification(ratings: Ratings): string; - - getKeyDimensions(): string[]; - - assessShortFormReadiness(ratings: Ratings): ShortFormAssessment; -} - -export function loadClassifier(url?: string): Promise; diff --git a/analysis/web/bicorder-classifier.js b/analysis/web/bicorder-classifier.js deleted file mode 100644 index ea85b94..0000000 --- a/analysis/web/bicorder-classifier.js +++ /dev/null @@ -1,335 +0,0 @@ -/** - * Bicorder Cluster Classifier - * - * Real-time protocol classification for the Bicorder web app. - * Predicts which protocol family (Relational/Cultural vs Institutional/Bureaucratic) - * a protocol belongs to based on dimension ratings. - * - * Usage: - * import { BicorderClassifier } from './bicorder-classifier.js'; - * - * const classifier = new BicorderClassifier(modelData); - * const result = classifier.predict(ratings); - * console.log(`Cluster: ${result.clusterName} (${result.confidence}% confidence)`); - */ - -export class BicorderClassifier { - /** - * @param {Object} model - Model data loaded from bicorder_model.json - * @param {string} bicorderVersion - Version of bicorder.json being used - * - * Simple version-matching approach: The model includes a bicorder_version - * field. When bicorder structure changes, update the version and retrain. - */ - constructor(model, bicorderVersion = null) { - this.model = model; - this.dimensions = model.dimensions; - this.keyDimensions = model.key_dimensions; - this.bicorderVersion = bicorderVersion; - - // Check version compatibility - if (bicorderVersion && model.bicorder_version && bicorderVersion !== model.bicorder_version) { - console.warn(`Model version (${model.bicorder_version}) doesn't match bicorder version (${bicorderVersion}). Results may be inaccurate.`); - } - } - - /** - * Standardize values using the fitted scaler - * @private - */ - _standardize(values) { - return values.map((val, i) => { - if (val === null || val === undefined) return null; - return (val - this.model.scaler.mean[i]) / this.model.scaler.scale[i]; - }); - } - - /** - * Calculate LDA score (position on discriminant axis) - * @private - */ - _ldaScore(scaledValues) { - // Fill missing values with 0 (mean in scaled space) - const filled = scaledValues.map(v => v === null ? 0 : v); - - // Calculate: coef · x + intercept - let score = this.model.lda.intercept; - for (let i = 0; i < filled.length; i++) { - score += this.model.lda.coefficients[i] * filled[i]; - } - return score; - } - - /** - * Calculate Euclidean distance - * @private - */ - _distance(a, b) { - let sum = 0; - for (let i = 0; i < a.length; i++) { - const diff = a[i] - b[i]; - sum += diff * diff; - } - return Math.sqrt(sum); - } - - /** - * Predict cluster for given ratings - * - * @param {Object} ratings - Map of dimension names to values (1-9) - * Can be partial - missing dimensions handled gracefully - * @param {Object} options - Options - * @param {boolean} options.detailed - Return detailed information (default: true) - * - * @returns {Object} Prediction result with: - * - cluster: Cluster number (1 or 2) - * - clusterName: Human-readable name - * - confidence: Confidence percentage (0-100) - * - completeness: Percentage of dimensions provided (0-100) - * - recommendedForm: 'short' or 'long' - * - ldaScore: Position on discriminant axis - * - distanceToBoundary: Distance from cluster boundary - */ - predict(ratings, options = { detailed: true }) { - // Convert ratings object to array - const values = this.dimensions.map(dim => ratings[dim] ?? null); - const providedCount = values.filter(v => v !== null).length; - const completeness = providedCount / this.dimensions.length; - - // Fill missing with neutral value (5 = middle of 1-9 scale) - const filled = values.map(v => v ?? 5); - - // Standardize - const scaled = this._standardize(filled); - - // Calculate LDA score - const ldaScore = this._ldaScore(scaled); - - // Predict cluster (LDA boundary at 0) - // Positive score = cluster 2 (Institutional) - // Negative score = cluster 1 (Relational) - const cluster = ldaScore > 0 ? 2 : 1; - const clusterName = this.model.cluster_names[cluster]; - - // Calculate confidence based on distance from boundary - const distanceToBoundary = Math.abs(ldaScore); - - // Confidence: higher when further from boundary - // Normalize based on typical strong separation (3.0) - let confidence = Math.min(1.0, distanceToBoundary / 3.0); - - // Adjust for completeness - const adjustedConfidence = confidence * (0.5 + 0.5 * completeness); - - // Recommend form - // Use long form when: - // 1. Low confidence (< 0.6) - // 2. Low completeness (< 50% of dimensions) - // 3. Near boundary (< 0.5 distance) - const shouldUseLongForm = - adjustedConfidence < this.model.thresholds.confidence_low || - completeness < this.model.thresholds.completeness_low || - distanceToBoundary < this.model.thresholds.boundary_distance_low; - - const recommendedForm = shouldUseLongForm ? 'long' : 'short'; - - const basicResult = { - cluster, - clusterName, - confidence: Math.round(adjustedConfidence * 100), - completeness: Math.round(completeness * 100), - recommendedForm, - }; - - if (!options.detailed) { - return basicResult; - } - - // Calculate distances to cluster centroids - const filledScaled = scaled.map(v => v ?? 0); - const distances = {}; - for (const [clusterId, centroid] of Object.entries(this.model.cluster_centroids_scaled)) { - distances[clusterId] = this._distance(filledScaled, centroid); - } - - // Count key dimensions provided - const keyDimensionsProvided = this.keyDimensions.filter( - dim => ratings[dim] !== null && ratings[dim] !== undefined - ).length; - - return { - ...basicResult, - ldaScore, - distanceToBoundary, - dimensionsProvided: providedCount, - dimensionsTotal: this.dimensions.length, - keyDimensionsProvided, - keyDimensionsTotal: this.keyDimensions.length, - distancesToCentroids: distances, - rawConfidence: Math.round(confidence * 100), - }; - } - - /** - * Get explanation of classification - * - * @param {Object} ratings - Dimension ratings - * @returns {string} Human-readable explanation - */ - explainClassification(ratings) { - const result = this.predict(ratings, { detailed: true }); - const lines = []; - - lines.push(`Protocol Classification: ${result.clusterName}`); - lines.push(`Confidence: ${result.confidence}%`); - lines.push(''); - - if (result.cluster === 2) { - lines.push('This protocol leans toward Institutional/Bureaucratic characteristics:'); - lines.push(' • More likely to be formal, standardized, top-down'); - lines.push(' • May involve state/corporate enforcement'); - lines.push(' • Tends toward precise, documented procedures'); - } else { - lines.push('This protocol leans toward Relational/Cultural characteristics:'); - lines.push(' • More likely to be emergent, community-based'); - lines.push(' • May involve voluntary participation'); - lines.push(' • Tends toward interpretive, flexible practices'); - } - - lines.push(''); - lines.push(`Distance from boundary: ${result.distanceToBoundary.toFixed(2)}`); - - if (result.distanceToBoundary < 0.5) { - lines.push('⚠️ This protocol is near the boundary between families.'); - lines.push(' It may exhibit characteristics of both types.'); - } - - lines.push(''); - lines.push(`Completeness: ${result.completeness}% (${result.dimensionsProvided}/${result.dimensionsTotal} dimensions)`); - - if (result.completeness < 100) { - lines.push('Note: Missing dimensions filled with neutral values (5)'); - lines.push(' Confidence improves with complete data'); - } - - lines.push(''); - lines.push(`Recommended form: ${result.recommendedForm.toUpperCase()}`); - - if (result.recommendedForm === 'long') { - lines.push('Reason: Use long form for:'); - if (result.confidence < 60) { - lines.push(' • Low classification confidence'); - } - if (result.completeness < 50) { - lines.push(' • Incomplete data'); - } - if (result.distanceToBoundary < 0.5) { - lines.push(' • Ambiguous positioning between families'); - } - } else { - lines.push(`Reason: High confidence classification with ${result.completeness}% data`); - } - - return lines.join('\n'); - } - - /** - * Get the list of key dimensions for short form - * @returns {Array} Dimension names - */ - getKeyDimensions() { - return [...this.keyDimensions]; - } - - /** - * Check if enough key dimensions are provided for reliable short-form classification - * @param {Object} ratings - Current ratings - * @returns {Object} Assessment with recommendation - */ - assessShortFormReadiness(ratings) { - const keyProvided = this.keyDimensions.filter( - dim => ratings[dim] !== null && ratings[dim] !== undefined - ); - - const coverage = keyProvided.length / this.keyDimensions.length; - const isReady = coverage >= 0.75; // 75% of key dimensions - - return { - ready: isReady, - keyDimensionsProvided: keyProvided.length, - keyDimensionsTotal: this.keyDimensions.length, - coverage: Math.round(coverage * 100), - missingKeyDimensions: this.keyDimensions.filter( - dim => !ratings[dim] - ), - }; - } -} - -/** - * Load model from JSON file - * - * @param {string} url - URL to bicorder_model.json - * @returns {Promise} Initialized classifier - */ -export async function loadClassifier(url = './bicorder_model.json') { - const response = await fetch(url); - const model = await response.json(); - return new BicorderClassifier(model); -} - -// Example usage (for testing in Node.js or browser console) -if (typeof window === 'undefined' && typeof module !== 'undefined') { - // Node.js example - const fs = require('fs'); - - function demo() { - const modelData = JSON.parse(fs.readFileSync('bicorder_model.json', 'utf8')); - const classifier = new BicorderClassifier(modelData); - - console.log('='.repeat(80)); - console.log('BICORDER CLASSIFIER - DEMO'); - console.log('='.repeat(80)); - - // Example 1: Community protocol - console.log('\nExample 1: Community-Based Protocol'); - console.log('-'.repeat(80)); - const communityRatings = { - 'Design_elite_vs_vernacular': 9, - 'Design_explicit_vs_implicit': 8, - 'Entanglement_flocking_vs_swarming': 9, - 'Entanglement_obligatory_vs_voluntary': 9, - 'Design_static_vs_malleable': 8, - }; - console.log(classifier.explainClassification(communityRatings)); - - // Example 2: Institutional protocol - console.log('\n\n' + '='.repeat(80)); - console.log('Example 2: Institutional Protocol'); - console.log('-'.repeat(80)); - const institutionalRatings = { - 'Design_elite_vs_vernacular': 1, - 'Design_explicit_vs_implicit': 1, - 'Entanglement_flocking_vs_swarming': 1, - 'Entanglement_obligatory_vs_voluntary': 1, - }; - console.log(classifier.explainClassification(institutionalRatings)); - - // Example 3: Check short form readiness - console.log('\n\n' + '='.repeat(80)); - console.log('Example 3: Short Form Readiness Assessment'); - console.log('-'.repeat(80)); - const partialRatings = { - 'Design_elite_vs_vernacular': 5, - 'Entanglement_flocking_vs_swarming': 6, - }; - const assessment = classifier.assessShortFormReadiness(partialRatings); - console.log(`Ready for reliable classification: ${assessment.ready}`); - console.log(`Key dimensions coverage: ${assessment.coverage}% (${assessment.keyDimensionsProvided}/${assessment.keyDimensionsTotal})`); - console.log(`Missing key dimensions: ${assessment.missingKeyDimensions.length}`); - } - - if (require.main === module) { - demo(); - } -} diff --git a/analysis/web/test-classifier.mjs b/analysis/web/test-classifier.mjs deleted file mode 100644 index 0318216..0000000 --- a/analysis/web/test-classifier.mjs +++ /dev/null @@ -1,41 +0,0 @@ -import { BicorderClassifier } from './bicorder-classifier.js'; -import { fileURLToPath } from 'url'; -import path from 'path'; -import fs from 'fs'; - -const __dirname = path.dirname(fileURLToPath(import.meta.url)); -const modelPath = path.join(__dirname, '..', 'bicorder_model.json'); -const modelData = JSON.parse(fs.readFileSync(modelPath, 'utf8')); -const classifier = new BicorderClassifier(modelData); - -console.log('='.repeat(80)); -console.log('BICORDER CLASSIFIER - TEST'); -console.log('='.repeat(80)); - -// Test 1 -console.log('\nTest 1: Institutional Protocol (e.g., Airport Security)'); -console.log('-'.repeat(80)); -const institutional = { - 'Design_elite_vs_vernacular': 1, - 'Design_explicit_vs_implicit': 1, - 'Entanglement_flocking_vs_swarming': 1, - 'Entanglement_obligatory_vs_voluntary': 1, -}; -const result1 = classifier.predict(institutional); -console.log(JSON.stringify(result1, null, 2)); - -// Test 2 -console.log('\n\nTest 2: Relational Protocol (e.g., Indigenous Practices)'); -console.log('-'.repeat(80)); -const relational = { - 'Design_elite_vs_vernacular': 9, - 'Entanglement_flocking_vs_swarming': 9, - 'Entanglement_obligatory_vs_voluntary': 9, -}; -const result2 = classifier.predict(relational); -console.log(JSON.stringify(result2, null, 2)); - -console.log('\n\n' + '='.repeat(80)); -console.log('✓ JavaScript classifier working correctly!'); -console.log(' Model size:', Math.round(fs.statSync(modelPath).size / 1024), 'KB'); -console.log('='.repeat(80));