#!/usr/bin/env node // ═══════════════════════════════════════════════════════════════════════════ // SIL Validation — Fråga 1: Hjälper SIL verkligen människor? // Mäter: tid till merge, regressionsbuggar, följsamhet, rätt vs fel // ═══════════════════════════════════════════════════════════════════════════ import { readFileSync, writeFileSync, existsSync, appendFileSync } from 'fs'; const DECISION_LOG = '/home/bernt/.openclaw/workspace/SIL/validation/decision-log.jsonl'; const BASELINE_LOG = '/home/bernt/.openclaw/workspace/SIL/validation/baseline-log.jsonl'; class HumanImpactValidator { constructor() { this.decisions = this.loadLog(DECISION_LOG); this.baselines = this.loadLog(BASELINE_LOG); } loadLog(path) { if (!existsSync(path)) return []; return readFileSync(path, 'utf8') .split('\n') .filter(line => line.trim()) .map(line => { try { return JSON.parse(line); } catch { return null; } }) .filter(Boolean); } /** * Logga ett beslut med utfall */ logDecision(decision) { const entry = { timestamp: new Date().toISOString(), pr: decision.pr, developer: decision.developer, // SIL:s rekommendation silRecommendation: decision.silRecommendation, silRiskScore: decision.silRiskScore, silConfidence: decision.silConfidence, // Utvecklarens beslut developerAction: decision.developerAction, // 'followed', 'ignored', 'modified' developerReason: decision.developerReason, // Utfall merged: decision.merged, mergeTime: decision.mergeTime, // timmar från PR till merge regressions: decision.regressions || [], // Utvärdering silWasRight: decision.silWasRight, // boolean eller null developerWasRight: decision.developerWasRight // boolean eller null }; appendFileSync(DECISION_LOG, JSON.stringify(entry) + '\n', 'utf8'); return entry; } /** * Beräkna baseline (utan SIL) */ calculateBaseline() { const baseline = { avgMergeTime: this.calculateAvgMergeTime(this.baselines), regressionRate: this.calculateRegressionRate(this.baselines), totalPRs: this.baselines.length }; return baseline; } /** * Beräkna metriker med SIL */ calculateWithSIL() { const withSil = { avgMergeTime: this.calculateAvgMergeTime(this.decisions), regressionRate: this.calculateRegressionRate(this.decisions), followRate: this.calculateFollowRate(), ignoreRate: this.calculateIgnoreRate(), silAccuracy: this.calculateSILAccuracy(), totalPRs: this.decisions.length }; return withSil; } calculateAvgMergeTime(entries) { const times = entries .map(e => e.mergeTime) .filter(t => t !== undefined && t !== null); if (times.length === 0) return null; return Math.round(times.reduce((a, b) => a + b, 0) / times.length * 10) / 10; } calculateRegressionRate(entries) { if (entries.length === 0) return null; const withRegressions = entries.filter(e => e.regressions && e.regressions.length > 0 ).length; return Math.round(withRegressions / entries.length * 100) + '%'; } calculateFollowRate() { if (this.decisions.length === 0) return null; const followed = this.decisions.filter(d => d.developerAction === 'followed' ).length; return Math.round(followed / this.decisions.length * 100) + '%'; } calculateIgnoreRate() { if (this.decisions.length === 0) return null; const ignored = this.decisions.filter(d => d.developerAction === 'ignored' ).length; return Math.round(ignored / this.decisions.length * 100) + '%'; } calculateSILAccuracy() { const evaluated = this.decisions.filter(d => d.silWasRight !== null); if (evaluated.length === 0) return null; const correct = evaluated.filter(d => d.silWasRight).length; return Math.round(correct / evaluated.length * 100) + '%'; } /** * Jämför SIL vs utvecklare */ compareSILvsDeveloper() { const comparisons = []; for (const d of this.decisions) { if (d.silWasRight === null || d.developerWasRight === null) continue; comparisons.push({ pr: d.pr, silRight: d.silWasRight, devRight: d.developerWasRight, scenario: d.silWasRight && d.developerWasRight ? 'both_right' : d.silWasRight && !d.developerWasRight ? 'sil_right_dev_wrong' : !d.silWasRight && d.developerWasRight ? 'dev_right_sil_wrong' : 'both_wrong' }); } const scenarios = { both_right: comparisons.filter(c => c.scenario === 'both_right').length, sil_right_dev_wrong: comparisons.filter(c => c.scenario === 'sil_right_dev_wrong').length, dev_right_sil_wrong: comparisons.filter(c => c.scenario === 'dev_right_sil_wrong').length, both_wrong: comparisons.filter(c => c.scenario === 'both_wrong').length }; return { total: comparisons.length, scenarios, silWinRate: comparisons.length > 0 ? Math.round(scenarios.sil_right_dev_wrong / comparisons.length * 100) + '%' : 'N/A', devWinRate: comparisons.length > 0 ? Math.round(scenarios.dev_right_sil_wrong / comparisons.length * 100) + '%' : 'N/A' }; } /** * Generera rapport */ generateReport() { const baseline = this.calculateBaseline(); const withSil = this.calculateWithSIL(); const comparison = this.compareSILvsDeveloper(); const report = { timestamp: new Date().toISOString(), baseline, withSil, comparison, // Förbättringar improvements: { mergeTime: baseline.avgMergeTime && withSil.avgMergeTime ? Math.round((baseline.avgMergeTime - withSil.avgMergeTime) / baseline.avgMergeTime * 100) + '%' : 'N/A', regressions: baseline.regressionRate && withSil.regressionRate ? Math.round((parseFloat(baseline.regressionRate) - parseFloat(withSil.regressionRate)) / parseFloat(baseline.regressionRate) * 100) + '%' : 'N/A' } }; this.printReport(report); return report; } printReport(report) { console.log('╔═══════════════════════════════════════════════════════════════╗'); console.log('║ VALIDATION: FRÅGA 1 ║'); console.log('║ Hjälper SIL verkligen människor? ║'); console.log('╚═══════════════════════════════════════════════════════════════╝'); console.log(); console.log('📊 BASELINE (utan SIL)'); console.log(); console.log(` PR:er: ${report.baseline.totalPRs}`); console.log(` Genomsnittlig tid till merge: ${report.baseline.avgMergeTime || 'N/A'} timmar`); console.log(` Regressionsfrekvens: ${report.baseline.regressionRate || 'N/A'}`); console.log(); console.log('📊 MED SIL'); console.log(); console.log(` PR:er: ${report.withSil.totalPRs}`); console.log(` Genomsnittlig tid till merge: ${report.withSil.avgMergeTime || 'N/A'} timmar`); console.log(` Regressionsfrekvens: ${report.withSil.regressionRate || 'N/A'}`); console.log(` Följsamhet: ${report.withSil.followRate || 'N/A'}`); console.log(` Ignorans: ${report.withSil.ignoreRate || 'N/A'}`); console.log(` SIL:s accuracy: ${report.withSil.silAccuracy || 'N/A'}`); console.log(); console.log('📈 FÖRBÄTTRINGAR'); console.log(); console.log(` Tid till merge: ${report.improvements.mergeTime}`); console.log(` Regressionsfel: ${report.improvements.regressions}`); console.log(); console.log('⚔️ SIL VS UTVECKLARE'); console.log(); console.log(` Totala jämförelser: ${report.comparison.total}`); console.log(` Båda har rätt: ${report.comparison.scenarios.both_right}`); console.log(` SIL rätt, utvecklare fel: ${report.comparison.scenarios.sil_right_dev_wrong}`); console.log(` Utvecklare rätt, SIL fel: ${report.comparison.scenarios.dev_right_sil_wrong}`); console.log(` Båda har fel: ${report.comparison.scenarios.both_wrong}`); console.log(); console.log(` SIL vinner: ${report.comparison.silWinRate}`); console.log(` Utvecklare vinner: ${report.comparison.devWinRate}`); console.log(); console.log('═══════════════════════════════════════════════════════════════\n'); } } // ── Main ────────────────────────────────────────────────────────────────── const validator = new HumanImpactValidator(); const command = process.argv[2] || '--report'; if (command === '--report') { validator.generateReport(); } else if (command === '--log') { // Exempel på att logga ett beslut const decision = { pr: process.argv[3] || 'PR-123', developer: process.argv[4] || 'developer@example.com', silRecommendation: process.argv[5] || 'APPROVE', silRiskScore: 25, silConfidence: 0.85, developerAction: process.argv[6] || 'followed', merged: true, mergeTime: 4.5, regressions: [], silWasRight: true, developerWasRight: true }; validator.logDecision(decision); console.log('✅ Beslut loggat'); } else { console.log('Användning:'); console.log(' node human-impact.mjs --report # Generera rapport'); console.log(' node human-impact.mjs --log # Logga beslut'); }