boc/SIL/stability.mjs

#!/usr/bin/env node
// ═══════════════════════════════════════════════════════════════════════════
// SIL Stability — Mät stabilitet och determinism
// Erik-krav: "Samma PR analyseras tio gånger. Blir analysen identisk?"
// ═══════════════════════════════════════════════════════════════════════════

import { execSync } from 'child_process';
import { readFileSync, writeFileSync, existsSync, appendFileSync } from 'fs';
import { createHash } from 'crypto';

const STABILITY_LOG = '/home/bernt/.openclaw/workspace/SIL/stability-log.jsonl';

class StabilityTest {
  constructor(repoPath) {
    this.repoPath = repoPath;
  }

  /**
   * Kör samma analys N gånger och jämför resultat
   */
  runStabilityTest(prBase = 'HEAD~1', prHead = 'HEAD', runs = 10) {
    console.log(`🔄 Kör stabilitetstest: ${runs} analyser av samma PR\n`);

    const results = [];
    const hashes = [];

    for (let i = 0; i < runs; i++) {
      const start = Date.now();

      try {
        const output = execSync(
          `node /home/bernt/.openclaw/workspace/SIL/pr-analyzer.mjs ${this.repoPath} ${prBase} ${prHead} 2>&1`,
          { encoding: 'utf8', maxBuffer: 10 * 1024 * 1024 }
        );

        // Extrahera nyckeldata från output
        const analysis = this.parseAnalysis(output);
        const hash = this.hashAnalysis(analysis);

        results.push({
          run: i + 1,
          duration: Date.now() - start,
          analysis,
          hash
        });

        hashes.push(hash);

        console.log(`  Körning ${i + 1}/${runs}: ${analysis.components.length} komponenter, hash=${hash.substring(0, 8)}`);
      } catch (e) {
        results.push({
          run: i + 1,
          duration: Date.now() - start,
          error: e.message
        });
        console.log(`  Körning ${i + 1}/${runs}: FEL — ${e.message}`);
      }
    }

    // Beräkna stabilitet
    const uniqueHashes = new Set(hashes);
    const stabilityScore = hashes.length > 0
      ? (hashes.length - uniqueHashes.size + 1) / hashes.length
      : 0;

    const determinismIndex = uniqueHashes.size === 1 ? 1.0 : 1.0 / uniqueHashes.size;

    // Jämför grafnoder mellan körningar
    const graphChurn = this.calculateGraphChurn(results);

    const report = {
      timestamp: new Date().toISOString(),
      pr: { base: prBase, head: prHead },
      runs,
      stability: {
        stabilityScore: Math.round(stabilityScore * 100) + '%',
        determinismIndex: Math.round(determinismIndex * 100) + '%',
        uniqueResults: uniqueHashes.size,
        totalRuns: hashes.length,
        isDeterministic: uniqueHashes.size === 1
      },
      graphChurn,
      perRun: results.map(r => ({
        run: r.run,
        duration: r.duration,
        components: r.analysis?.components?.length || 0,
        hash: r.hash?.substring(0, 8) || 'error'
      })),
      // Erik: "Om analysen varierar kraftigt trots samma indata finns ett determinismproblem"
      verdict: uniqueHashes.size === 1
        ? '✅ DETERMINISTISK — Samma indata ger samma utdata'
        : uniqueHashes.size <= 2
          ? '⚠️  MARGINELL VARIATION — Acceptabelt men undersök'
          : '🔴 ICKE-DETERMINISTISK — Kritiskt problem'
    };

    this.logResult(report);
    this.printReport(report);

    return report;
  }

  /**
   * Extrahera strukturerad data från analys-output
   */
  parseAnalysis(output) {
    const components = [];
    const lines = output.split('\n');

    for (const line of lines) {
      // Extrahera komponenter
      const match = line.match(/✅\s+(.+)/);
      if (match && !line.includes('Körning')) {
        components.push(match[1].trim());
      }
    }

    return { components };
  }

  /**
   * Hasha analysresultat för jämförelse
   */
  hashAnalysis(analysis) {
    return createHash('sha256')
      .update(JSON.stringify(analysis.components.sort()))
      .digest('hex');
  }

  /**
   * Beräkna graf-churn mellan körningar
   */
  calculateGraphChurn(results) {
    const validResults = results.filter(r => r.analysis && !r.error);
    if (validResults.length < 2) return null;

    // Jämför första och sista körningen
    const first = validResults[0].analysis.components;
    const last = validResults[validResults.length - 1].analysis.components;

    const firstSet = new Set(first);
    const lastSet = new Set(last);

    const added = [...lastSet].filter(c => !firstSet.has(c));
    const removed = [...firstSet].filter(c => !lastSet.has(c));

    return {
      nodeChurn: added.length + removed.length,
      added,
      removed,
      stable: added.length === 0 && removed.length === 0
    };
  }

  logResult(report) {
    appendFileSync(STABILITY_LOG, JSON.stringify(report) + '\n', 'utf8');
  }

  printReport(report) {
    console.log('\n═══════════════════════════════════════════════════════════════');
    console.log('           SIL STABILITY REPORT');
    console.log('═══════════════════════════════════════════════════════════════\n');

    console.log(`📊 STABILITET\n`);
    console.log(`   Stability Score:    ${report.stability.stabilityScore}`);
    console.log(`   Determinism Index:  ${report.stability.determinismIndex}`);
    console.log(`   Unika resultat:     ${report.stability.uniqueResults}/${report.stability.totalRuns}`);
    console.log();

    console.log(`🔍 VERDIKT\n`);
    console.log(`   ${report.verdict}\n`);

    if (report.graphChurn) {
      console.log(`📈 GRAPH CHURN\n`);
      console.log(`   Noder ändrade: ${report.graphChurn.nodeChurn}`);
      if (report.graphChurn.added.length > 0) {
        console.log(`   Tillagda: ${report.graphChurn.added.join(', ')}`);
      }
      if (report.graphChurn.removed.length > 0) {
        console.log(`   Borttagna: ${report.graphChurn.removed.join(', ')}`);
      }
      console.log(`   Stabil: ${report.graphChurn.stable ? 'Ja' : 'Nej'}\n`);
    }

    console.log(`⏱️  PER KÖRNING\n`);
    for (const run of report.perRun) {
      const icon = run.hash === report.perRun[0].hash ? '✅' : '⚠️';
      console.log(`   ${icon} Körning ${run.run}: ${run.duration}ms, ${run.components} komponenter, hash=${run.hash}`);
    }
    console.log();

    console.log('═══════════════════════════════════════════════════════════════\n');
  }

  /**
   * Kör stabilitetstest på historiska PR:er
   */
  runHistoricalStabilityTest(count = 10) {
    console.log(`📚 Kör stabilitetstest på ${count} historiska PR:er...\n`);

    try {
      const merges = execSync(
        `git log --merges --pretty=format:"%H %P" -n ${count}`,
        { cwd: this.repoPath, encoding: 'utf8' }
      ).trim().split('\n');

      for (const merge of merges) {
        const [hash, ...parents] = merge.split(' ');
        if (parents.length >= 2) {
          console.log(`\n🔍 Testar PR: ${hash.substring(0, 8)}`);
          this.runStabilityTest(parents[0], parents[1], 3); // 3 körningar per PR
        }
      }
    } catch (e) {
      console.error('Fel:', e.message);
    }
  }
}

// ── Main ──────────────────────────────────────────────────────────────────

const repoPath = process.argv[2] || '/home/bernt/repos/quixzoom.com';
const command = process.argv[3] || '--test';
const test = new StabilityTest(repoPath);

if (command === '--test') {
  const runs = parseInt(process.argv[4]) || 10;
  test.runStabilityTest('HEAD~1', 'HEAD', runs);
} else if (command === '--historical') {
  const count = parseInt(process.argv[4]) || 10;
  test.runHistoricalStabilityTest(count);
} else {
  console.log('Användning:');
  console.log('  node stability.mjs [repo] --test [runs]      # Kör stabilitetstest');
  console.log('  node stability.mjs [repo] --historical [n]   # Testa historiska PR:er');
}