boc/quixzoom-capture-pipeline/replay/full-replay.js

/**
 * QUIXZOOM Full Replay - End-to-end test mot PostgreSQL
 * 
 * Kör hela kedjan:
 * PostgreSQL → Object Identity Engine → UKG → Evidence Layer → Temporal Layer
 */

const { Pool } = require('pg');
const ObjectIdentityPipeline = require('../identity-engine/pipeline');

class FullReplay {
  constructor() {
    this.db = new Pool({
      host: 'localhost',
      port: 5433,
      user: 'quixzoom',
      password: 'quixzoom2026',
      database: 'quixzoom',
    });
    
    this.oie = new ObjectIdentityPipeline({
      autoMergeThreshold: 0.50,
      waitThreshold: 0.30,
    });
    
    this.metrics = {
      startTime: Date.now(),
      eventsProcessed: 0,
      objectsCreated: 0,
      objectsMerged: 0,
      dbReads: 0,
      dbWrites: 0,
      latencies: [],
      errors: [],
    };
  }

  async run() {
    console.log('[REPLAY] Starting full replay against PostgreSQL...\n');
    
    // 1. Ladda alla observationer från databasen
    console.log('[REPLAY] Loading observations from PostgreSQL...');
    const startLoad = Date.now();
    const observations = await this.loadObservations();
    const loadTime = Date.now() - startLoad;
    console.log(`[REPLAY] Loaded ${observations.length} observations in ${loadTime}ms`);
    
    // 2. Kör replay
    console.log('[REPLAY] Processing observations through OIE...\n');
    const startProcess = Date.now();
    
    for (let i = 0; i < observations.length; i++) {
      const obs = observations[i];
      const eventStart = Date.now();
      
      try {
        // Konvertera DB-format till OIE-format
        const oieObservation = {
          id: obs.observation_id,
          objectType: obs.source_type === 'zoomer' ? 'street_lamp' : 'unknown',
          location: {
            lat: obs.latitude,
            lng: obs.longitude,
          },
          gpsAccuracy: obs.gps_accuracy || 3,
          timestamp: obs.created_at,
          attributes: obs.ai_analysis?.attributes || {},
        };
        
        // Processa genom OIE
        const result = await this.oie.process(oieObservation);
        
        if (result.action === 'new_object') {
          this.metrics.objectsCreated++;
        } else if (result.action === 'merge') {
          this.metrics.objectsMerged++;
        }
        
        this.metrics.eventsProcessed++;
        this.metrics.latencies.push(Date.now() - eventStart);
        
        // Logga var 1000:e
        if ((i + 1) % 1000 === 0) {
          this.logProgress(i + 1, observations.length);
        }
        
      } catch (error) {
        this.metrics.errors.push({
          observation: obs.observation_id,
          error: error.message,
        });
      }
    }
    
    const processTime = Date.now() - startProcess;
    
    // 3. Sammanfattning
    console.log('\n=== REPLAY COMPLETE ===');
    this.printSummary(processTime);
    
    await this.db.end();
  }

  async loadObservations() {
    const result = await this.db.query(`
      SELECT observation_id, source_type, source_id, 
             latitude, longitude, gps_accuracy, 
             ai_analysis, created_at
      FROM observations
      ORDER BY created_at
      LIMIT 10000
    `);
    this.metrics.dbReads++;
    return result.rows;
  }

  logProgress(current, total) {
    const elapsed = Date.now() - this.metrics.startTime;
    const rate = current / (elapsed / 1000);
    const percent = ((current / total) * 100).toFixed(1);
    
    process.stdout.write(`\r[REPLAY] ${percent}% | ${current}/${total} | ${rate.toFixed(0)} events/s | Objects: ${this.oie.objects.size}`);
  }

  printSummary(processTime) {
    const totalTime = Date.now() - this.metrics.startTime;
    const avgLatency = this.metrics.latencies.reduce((a, b) => a + b, 0) / this.metrics.latencies.length;
    const sortedLatencies = [...this.metrics.latencies].sort((a, b) => a - b);
    const p99 = sortedLatencies[Math.floor(sortedLatencies.length * 0.99)];
    
    console.log(`\nTotal time: ${totalTime}ms`);
    console.log(`Processing time: ${processTime}ms`);
    console.log(`Events processed: ${this.metrics.eventsProcessed}`);
    console.log(`Events/sec: ${(this.metrics.eventsProcessed / (processTime / 1000)).toFixed(0)}`);
    console.log(`Objects created: ${this.metrics.objectsCreated}`);
    console.log(`Objects merged: ${this.metrics.objectsMerged}`);
    console.log(`Final object count: ${this.oie.objects.size}`);
    console.log(`Avg latency: ${avgLatency.toFixed(2)}ms`);
    console.log(`P99 latency: ${p99}ms`);
    console.log(`Errors: ${this.metrics.errors.length}`);
    
    if (this.metrics.errors.length > 0) {
      console.log('\nFirst 5 errors:');
      this.metrics.errors.slice(0, 5).forEach(e => console.log(`  ${e.observation}: ${e.error}`));
    }
  }
}

// Kör
const replay = new FullReplay();
replay.run().catch(console.error);