/** * QUIXZOOM Benchmark Suite * * Automatisk testning av OIE och Change Detection. * Kör efter varje ändring och genererar rapport. * * Exempel på output: * Metric Version A Version B * Precision 96.1% 97.4% * Recall 94.8% 96.2% * False merges 41 19 * Missed merges 112 73 * Avg latency 34 ms 28 ms */ class BenchmarkSuite { constructor(options = {}) { this.groundTruth = new Map(); // observationId -> expectedObjectId this.results = []; this.metrics = { precision: 0, recall: 0, falseMerges: 0, missedMerges: 0, avgLatency: 0, throughput: 0, }; } /** * ============================================================ * LADDA GROUND TRUTH * ============================================================ */ loadGroundTruth(annotations) { // annotations: [{ observationId, expectedObjectId, isSameObject }] for (const ann of annotations) { this.groundTruth.set(ann.observationId, ann); } console.log(`[BENCH] Loaded ${annotations.length} ground truth annotations`); } generateGroundTruthFromSimulator(simulator) { // Generera ground truth från simulatorn // Objekt som skapats från samma "fysiska" objekt bör matchas const annotations = []; for (const [objectId, obj] of simulator.objects) { // Hitta alla observationer för detta objekt const observations = simulator.observations.filter(o => o.objectId === objectId); for (const obs of observations) { annotations.push({ observationId: obs.id, expectedObjectId: objectId, isSameObject: true, }); } } this.loadGroundTruth(annotations); return annotations; } /** * ============================================================ * KÖR BENCHMARK * ============================================================ */ async run(engine, events, options = {}) { console.log('[BENCH] Starting benchmark...'); const startTime = Date.now(); const results = { version: options.version || 'unknown', engine: engine.constructor.name, totalEvents: events.length, processedEvents: 0, correctMatches: 0, falseMatches: 0, missedMatches: 0, newObjects: 0, mergedObjects: 0, latencies: [], errors: [], }; // Kör varje event for (let i = 0; i < events.length; i++) { const event = events[i]; const eventStart = Date.now(); try { if (event.type === 'observation_created') { const result = await engine.process({ id: event.payload.observationId, objectType: event.payload.objectType, location: event.payload.location, timestamp: event.payload.timestamp, attributes: event.payload.attributes, }); const latency = Date.now() - eventStart; results.latencies.push(latency); // Utvärdera mot ground truth const groundTruth = this.groundTruth.get(event.payload.observationId); if (groundTruth) { if (result.action === 'merge' || result.action === 'new') { if (result.objectId === groundTruth.expectedObjectId) { results.correctMatches++; } else { results.falseMatches++; } } } if (result.action === 'merge') results.mergedObjects++; if (result.action === 'new') results.newObjects++; } results.processedEvents++; } catch (error) { results.errors.push({ eventId: event.id, error: error.message, }); } // Progress if (i % 1000 === 0) { console.log(`[BENCH] Processed ${i}/${events.length} events...`); } } const totalTime = Date.now() - startTime; // Beräkna metrics const totalObservations = results.correctMatches + results.falseMatches + results.missedMatches; results.precision = totalObservations > 0 ? (results.correctMatches / (results.correctMatches + results.falseMatches)) : 0; results.recall = totalObservations > 0 ? (results.correctMatches / totalObservations) : 0; results.avgLatency = results.latencies.length > 0 ? (results.latencies.reduce((a, b) => a + b, 0) / results.latencies.length) : 0; results.throughput = totalTime > 0 ? (results.processedEvents / (totalTime / 1000)) : 0; results.totalTime = totalTime; this.results.push(results); console.log('[BENCH] Benchmark complete'); return results; } /** * ============================================================ * JÄMFÖR VERSIONER * ============================================================ */ compare(versionA, versionB) { const a = this.results.find(r => r.version === versionA); const b = this.results.find(r => r.version === versionB); if (!a || !b) { throw new Error('Version not found in results'); } return { versionA: a, versionB: b, comparison: { precision: { a: (a.precision * 100).toFixed(1) + '%', b: (b.precision * 100).toFixed(1) + '%', winner: a.precision > b.precision ? 'A' : 'B', delta: ((b.precision - a.precision) * 100).toFixed(1) + '%', }, recall: { a: (a.recall * 100).toFixed(1) + '%', b: (b.recall * 100).toFixed(1) + '%', winner: a.recall > b.recall ? 'A' : 'B', delta: ((b.recall - a.recall) * 100).toFixed(1) + '%', }, falseMerges: { a: a.falseMatches, b: b.falseMatches, winner: a.falseMatches < b.falseMatches ? 'A' : 'B', delta: (b.falseMatches - a.falseMatches), }, missedMerges: { a: a.missedMatches || 0, b: b.missedMatches || 0, winner: (a.missedMatches || 0) < (b.missedMatches || 0) ? 'A' : 'B', delta: (b.missedMatches || 0) - (a.missedMatches || 0), }, avgLatency: { a: a.avgLatency.toFixed(0) + ' ms', b: b.avgLatency.toFixed(0) + ' ms', winner: a.avgLatency < b.avgLatency ? 'A' : 'B', delta: (b.avgLatency - a.avgLatency).toFixed(0) + ' ms', }, throughput: { a: a.throughput.toFixed(0) + ' events/s', b: b.throughput.toFixed(0) + ' events/s', winner: a.throughput > b.throughput ? 'A' : 'B', delta: (b.throughput - a.throughput).toFixed(0) + ' events/s', }, }, }; } /** * ============================================================ * RAPPORT * ============================================================ */ generateReport() { console.log('\n=== BENCHMARK REPORT ===\n'); for (const result of this.results) { console.log(`Version: ${result.version}`); console.log(` Engine: ${result.engine}`); console.log(` Events: ${result.processedEvents}/${result.totalEvents}`); console.log(` Precision: ${(result.precision * 100).toFixed(1)}%`); console.log(` Recall: ${(result.recall * 100).toFixed(1)}%`); console.log(` False merges: ${result.falseMatches}`); console.log(` New objects: ${result.newObjects}`); console.log(` Merged objects: ${result.mergedObjects}`); console.log(` Avg latency: ${result.avgLatency.toFixed(0)} ms`); console.log(` Throughput: ${result.throughput.toFixed(0)} events/s`); console.log(` Total time: ${(result.totalTime / 1000).toFixed(1)} s`); console.log(` Errors: ${result.errors.length}`); console.log(); } if (this.results.length >= 2) { const comparison = this.compare( this.results[0].version, this.results[1].version ); console.log('=== COMPARISON ===\n'); console.log('Metric Version A Version B Winner Delta'); console.log('─────────────────────────────────────────────────────────────────────'); const metrics = ['precision', 'recall', 'falseMerges', 'missedMerges', 'avgLatency', 'throughput']; for (const metric of metrics) { const comp = comparison.comparison[metric]; console.log( `${metric.padEnd(18)} ${comp.a.padEnd(16)} ${comp.b.padEnd(16)} ${comp.winner.padEnd(9)} ${comp.delta}` ); } } } exportToFile(path) { const fs = require('fs'); fs.writeFileSync(path, JSON.stringify({ results: this.results, generatedAt: new Date().toISOString(), }, null, 2)); console.log(`[BENCH] Report exported to ${path}`); } } // Exportera module.exports = BenchmarkSuite; // Demo if (require.main === module) { const ObjectIdentityPipeline = require('../identity-engine/pipeline'); const CitySimulator = require('../test-data/simulator'); async function runBenchmark() { console.log('=== BENCHMARK SUITE DEMO ===\n'); // 1. Generera testdata console.log('Generating test data...'); const simulator = new CitySimulator('Stockholm', { north: 59.4, south: 59.2, east: 18.2, west: 17.9, }); simulator.generate(); // 2. Generera ground truth (förenklad: använd subset) const benchmark = new BenchmarkSuite(); const subsetObservations = simulator.observations.slice(0, 1000); const annotations = subsetObservations.map(obs => ({ observationId: obs.id, expectedObjectId: obs.objectId, isSameObject: true, })); benchmark.loadGroundTruth(annotations); // 3. Skapa events från observationer const events = subsetObservations.map(obs => ({ id: `evt_${obs.id}`, type: 'observation_created', timestamp: obs.timestamp.getTime(), payload: { observationId: obs.id, objectType: obs.objectType, location: obs.location, timestamp: obs.timestamp.getTime(), attributes: obs.attributes, }, })); // 4. Kör Version A (default weights) console.log('\n--- Running Version A (default weights) ---'); const engineA = new ObjectIdentityPipeline(); const resultA = await benchmark.run(engineA, events, { version: 'A' }); // 5. Kör Version B (adjusted weights) console.log('\n--- Running Version B (spatial-heavy weights) ---'); const engineB = new ObjectIdentityPipeline({ spatialWeight: 0.5, visualWeight: 0.25, contextWeight: 0.15, temporalWeight: 0.10, }); const resultB = await benchmark.run(engineB, events, { version: 'B' }); // 6. Generera rapport benchmark.generateReport(); // 7. Exportera benchmark.exportToFile('/tmp/benchmark-report.json'); } runBenchmark().catch(console.error); }