boc/quixzoom-capture-pipeline/pilot/ai-benchmark-suite.js

/**
 * QUIXZOOM AI Benchmark Suite
 * 
 * Jämför modeller på samma data:
 * - Cloud Vision
 * - YOLOv8
 * - Grounding DINO
 * - Egen modell (senare)
 */

const fs = require('fs');
const path = require('path');

class AIBenchmarkSuite {
  constructor() {
    this.results = {
      cloudVision: [],
      yolo: [],
      groundingDino: [],
      custom: []
    };
    
    this.metrics = {
      precision: {},
      recall: {},
      costPerImage: {},
      latency: {}
    };
  }

  /**
   * ============================================================
   * HUVUDMETOD: Kör benchmark på dataset
   * ============================================================
   */

  async runBenchmark(imagePaths, options = {}) {
    console.log('╔════════════════════════════════════════════════════════════╗');
    console.log('║     AI BENCHMARK SUITE                                    ║');
    console.log('╚════════════════════════════════════════════════════════════╝\n');
    
    const regions = options.regions || ['bangkok', 'torrevieja', 'stockholm'];
    
    console.log(`Dataset: ${imagePaths.length} bilder`);
    console.log(`Regioner: ${regions.join(', ')}\n`);
    
    // 1. Cloud Vision
    console.log('=== 1. CLOUD VISION ===');
    const cvResults = await this.benchmarkCloudVision(imagePaths);
    this.results.cloudVision = cvResults;
    
    // 2. YOLOv8
    console.log('\n=== 2. YOLOv8 ===');
    const yoloResults = await this.benchmarkYOLO(imagePaths);
    this.results.yolo = yoloResults;
    
    // 3. Grounding DINO (om tillgänglig)
    console.log('\n=== 3. GROUNDING DINO ===');
    const dinoResults = await this.benchmarkGroundingDINO(imagePaths);
    this.results.groundingDino = dinoResults;
    
    // Sammanställ rapport
    return this.generateReport();
  }

  /**
   * ============================================================
   * BENCHMARK: Cloud Vision
   * ============================================================
   */

  async benchmarkCloudVision(imagePaths) {
    const vision = require('@google-cloud/vision');
    const client = new vision.ImageAnnotatorClient();
    
    const results = [];
    const startTime = Date.now();
    
    for (let i = 0; i < imagePaths.length; i++) {
      const path = imagePaths[i];
      const imgStart = Date.now();
      
      try {
        const [result] = await client.objectLocalization(path);
        const objects = result.localizedObjectAnnotations || [];
        
        results.push({
          image: path,
          detections: objects.map(obj => ({
            label: obj.name,
            confidence: obj.score,
            bbox: obj.boundingPoly
          })),
          latency: Date.now() - imgStart,
          timestamp: new Date().toISOString()
        });
        
        if ((i + 1) % 10 === 0) {
          console.log(`  ${i + 1}/${imagePaths.length} bilder...`);
        }
        
      } catch (error) {
        results.push({
          image: path,
          error: error.message,
          latency: Date.now() - imgStart
        });
      }
    }
    
    const totalTime = Date.now() - startTime;
    
    return {
      model: 'Cloud Vision',
      totalImages: imagePaths.length,
      totalTime,
      avgLatency: totalTime / imagePaths.length,
      detections: results.reduce((sum, r) => sum + (r.detections?.length || 0), 0),
      results
    };
  }

  /**
   * ============================================================
   * BENCHMARK: YOLOv8
   * ============================================================
   */

  async benchmarkYOLO(imagePaths) {
    // Kör Python-skript för YOLO
    const { execSync } = require('child_process');
    
    const results = [];
    const startTime = Date.now();
    
    for (let i = 0; i < Math.min(imagePaths.length, 100); i++) {
      const path = imagePaths[i];
      const imgStart = Date.now();
      
      try {
        // Kör YOLO via Python
        const output = execSync(
          `python3 -c "
            from ultralytics import YOLO
            model = YOLO('yolov8n.pt')
            results = model('${path}', verbose=False)
            boxes = results[0].boxes
            print(f'DETECTIONS:{len(boxes)}')
            for box in boxes:
              cls = int(box.cls)
              conf = float(box.conf)
              print(f'{model.names[cls]},{conf:.3f}')
          "`,
          { encoding: 'utf8', timeout: 30000 }
        );
        
        // Parsa output
        const lines = output.trim().split('\n');
        const detectionCount = parseInt(lines[0].split(':')[1]) || 0;
        const detections = lines.slice(1).map(line => {
          const [label, confidence] = line.split(',');
          return { label, confidence: parseFloat(confidence) };
        });
        
        results.push({
          image: path,
          detections,
          latency: Date.now() - imgStart
        });
        
      } catch (error) {
        results.push({
          image: path,
          error: error.message,
          latency: Date.now() - imgStart
        });
      }
    }
    
    const totalTime = Date.now() - startTime;
    
    return {
      model: 'YOLOv8',
      totalImages: results.length,
      totalTime,
      avgLatency: totalTime / results.length,
      detections: results.reduce((sum, r) => sum + (r.detections?.length || 0), 0),
      results
    };
  }

  /**
   * ============================================================
   * BENCHMARK: Grounding DINO
   * ============================================================
   */

  async benchmarkGroundingDINO(imagePaths) {
    // Grounding DINO har kompatibilitetsproblem just nu
    console.log('  ⚠️  Grounding DINO ej tillgänglig (CUDA-kompatibilitet)');
    
    return {
      model: 'Grounding DINO',
      totalImages: 0,
      totalTime: 0,
      avgLatency: 0,
      detections: 0,
      error: 'CUDA-kompatibilitet',
      results: []
    };
  }

  /**
   * ============================================================
   * RAPPORT
   * ============================================================
   */

  generateReport() {
    console.log('\n╔════════════════════════════════════════════════════════════╗');
    console.log('║     BENCHMARK RAPPORT                                     ║');
    console.log('╚════════════════════════════════════════════════════════════╝\n');
    
    const report = {
      timestamp: new Date().toISOString(),
      models: {}
    };
    
    for (const [modelName, result] of Object.entries(this.results)) {
      if (result.totalImages > 0) {
        report.models[modelName] = {
          precision: 'TBD (kräver gold labels)',
          recall: 'TBD (kräver gold labels)',
          costPerImage: this.estimateCost(modelName),
          avgLatency: `${result.avgLatency?.toFixed(0)}ms`,
          totalDetections: result.detections,
          avgDetectionsPerImage: (result.detections / result.totalImages).toFixed(2)
        };
        
        console.log(`=== ${result.model} ===`);
        console.log(`  Bilder: ${result.totalImages}`);
        console.log(`  Detektioner: ${result.detections}`);
        console.log(`  Genomsnitt/bild: ${report.models[modelName].avgDetectionsPerImage}`);
        console.log(`  Latens: ${report.models[modelName].avgLatency}`);
        console.log(`  Kostnad/bild: ${report.models[modelName].costPerImage}`);
        console.log(`  Precision: ${report.models[modelName].precision}`);
        console.log();
      }
    }
    
    // Spara rapport
    const reportPath = `/tmp/ai-benchmark-${Date.now()}.json`;
    fs.writeFileSync(reportPath, JSON.stringify(report, null, 2));
    console.log(`Rapport sparad: ${reportPath}`);
    
    return report;
  }

  estimateCost(modelName) {
    const costs = {
      cloudVision: '$0.0015',
      yolo: '$0 (lokal)',
      groundingDino: '$0 (lokal)',
      custom: '$0 (lokal)'
    };
    return costs[modelName] || 'Okänd';
  }

  /**
   * ============================================================
   * SPARA ALLA RESULTAT
   * ============================================================
   */

  saveAllResults(outputDir) {
    fs.mkdirSync(outputDir, { recursive: true });
    
    for (const [modelName, results] of Object.entries(this.results)) {
      const filePath = path.join(outputDir, `${modelName}-results.json`);
      fs.writeFileSync(filePath, JSON.stringify(results, null, 2));
      console.log(`Sparade: ${filePath}`);
    }
    
    // Metadata
    const metadata = {
      timestamp: new Date().toISOString(),
      models: Object.keys(this.results),
      totalImages: this.results.cloudVision.totalImages || 0,
      version: '1.0'
    };
    
    fs.writeFileSync(
      path.join(outputDir, 'metadata.json'),
      JSON.stringify(metadata, null, 2)
    );
  }
}

module.exports = AIBenchmarkSuite;

// Demo
if (require.main === module) {
  const suite = new AIBenchmarkSuite();
  
  console.log('╔════════════════════════════════════════════════════════════╗');
  console.log('║     AI BENCHMARK SUITE — DEMO                             ║');
  console.log('╚════════════════════════════════════════════════════════════╝\n');
  
  console.log('Användning:');
  console.log('  const suite = new AIBenchmarkSuite();');
  console.log('  const report = await suite.runBenchmark(imagePaths);');
  console.log('');
  console.log('Sparar alla resultat:');
  console.log('  suite.saveAllResults("/data/ai-results/");');
  console.log('');
  console.log('✅ Benchmark Suite redo!');
}