boc/quixzoom-capture-pipeline/test-data/export-to-postgres.js

/**
 * QUIXZOOM Test Data Export to PostgreSQL
 *
 * Exporterar simulerad stad till PostgreSQL med batching.
 */

const { Pool } = require('pg');
const SyntheticCity = require('./synthetic-city');

class PostgresExporter {
  constructor(connectionConfig) {
    this.pool = new Pool(connectionConfig);
    this.batchSize = 1000;
  }

  async export(city) {
    console.log('[EXPORT] Starting export to PostgreSQL...');
    const startTime = Date.now();

    try {
      // 1. Zoomers
      await this.exportZoomers(city.zoomers);

      // 2. Objects
      await this.exportObjects(city.objects);

      // 3. Observations
      await this.exportObservations(city.observations);

      // 4. Evidence (subset)
      await this.exportEvidence(city.observations.slice(0, 100000));

      // 5. Events (subset)
      await this.exportEvents(city.observations.slice(0, 50000));

      const duration = Date.now() - startTime;
      console.log(`[EXPORT] Complete in ${duration}ms`);

      // Verifiera
      await this.verify();

    } catch (error) {
      console.error('[EXPORT] Error:', error.message);
      throw error;
    }
  }

  async exportZoomers(zoomers) {
    console.log(`[EXPORT] Exporting ${zoomers.length} zoomers...`);

    const columns = ['id', 'name', 'email', 'city', 'level', 'rating', 'missions_completed', 'joined_at', 'device', 'status'];

    for (let i = 0; i < zoomers.length; i += this.batchSize) {
      const batch = zoomers.slice(i, i + this.batchSize);
      const values = batch.map((z, idx) => {
        const offset = idx * columns.length;
        return `(${columns.map((_, ci) => `$${offset + ci + 1}`).join(', ')})`;
      }).join(', ');

      const flatValues = batch.flatMap(z => [
        z.id, z.name || `Zoomer ${z.id}`, z.email || `${z.id}@example.com`, z.city || 'Stockholm',
        z.level, z.rating || 4.0, z.missionsCompleted || 0, new Date(),
        JSON.stringify(z.device), 'active',
      ]);

      await this.pool.query(`
        INSERT INTO zoomers (${columns.join(', ')})
        VALUES ${values}
        ON CONFLICT (id) DO NOTHING
      `, flatValues);

      console.log(`[EXPORT] Zoomers: ${Math.min(i + this.batchSize, zoomers.length)}/${zoomers.length}`);
    }
  }

  async exportObjects(objects) {
    console.log(`[EXPORT] Exporting ${objects.length} objects...`);

    for (let i = 0; i < objects.length; i += this.batchSize) {
      const batch = objects.slice(i, i + this.batchSize);
      const values = batch.map((o, idx) => {
        const offset = idx * 10;
        return `($${offset + 1}, $${offset + 2}, ST_SetSRID(ST_MakePoint($${offset + 3}, $${offset + 4}), 4326), $${offset + 5}, $${offset + 6}, $${offset + 7}, $${offset + 8}, $${offset + 9}, $${offset + 10}, $${offset + 11})`;
      }).join(', ');

      const flatValues = batch.flatMap(o => [
        o.id, o.type, o.location.lng, o.location.lat, 15 + Math.random() * 35,
        JSON.stringify(o.attributes), new Date(2023, 0, 1), new Date(),
        0, 0.8 + Math.random() * 0.2, 'ok',
      ]);

      await this.pool.query(`
        INSERT INTO objects (id, type, location, altitude, attributes, first_seen, last_seen, observation_count, health, status)
        VALUES ${values}
        ON CONFLICT (id) DO NOTHING
      `, flatValues);

      if (i % 10000 === 0) {
        console.log(`[EXPORT] Objects: ${Math.min(i + this.batchSize, objects.length)}/${objects.length}`);
      }
    }
  }

  async exportObservations(observations) {
    console.log(`[EXPORT] Exporting ${observations.length} observations...`);

    for (let i = 0; i < observations.length; i += this.batchSize) {
      const batch = observations.slice(i, i + this.batchSize);
      const values = batch.map((o, idx) => {
        const offset = idx * 10;
        return `($${offset + 1}, $${offset + 2}, $${offset + 3}, ST_SetSRID(ST_MakePoint($${offset + 4}, $${offset + 5}), 4326), $${offset + 6}, $${offset + 7}, $${offset + 8}, $${offset + 9}, $${offset + 10}, $${offset + 11})`;
      }).join(', ');

      const flatValues = batch.flatMap(o => [
        o.id, o.objectId, o.zoomerId || 'zoomer_001', o.location.lng, o.location.lat,
        15 + Math.random() * 35, o.gpsAccuracy || 3, o.timestamp || new Date(),
        JSON.stringify(o.attributes), JSON.stringify(o.quality || {}), o.weather || 'clear',
      ]);

      await this.pool.query(`
        INSERT INTO observations (id, object_id, zoomer_id, location, altitude, gps_accuracy, timestamp, attributes, quality, weather)
        VALUES ${values}
        ON CONFLICT (id) DO NOTHING
      `, flatValues);

      if (i % 50000 === 0) {
        console.log(`[EXPORT] Observations: ${Math.min(i + this.batchSize, observations.length)}/${observations.length}`);
      }
    }
  }

  async exportEvidence(observations) {
    console.log(`[EXPORT] Exporting evidence for ${observations.length} observations...`);

    for (let i = 0; i < observations.length; i += this.batchSize) {
      const batch = observations.slice(i, i + this.batchSize);
      const values = batch.map((o, idx) => {
        const offset = idx * 7;
        return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4}, $${offset + 5}, $${offset + 6}, $${offset + 7})`;
      }).join(', ');

      const flatValues = batch.flatMap(o => [
        `ev_${o.id}`, o.id, o.objectId, 'image', o.zoomerId || 'zoomer_001',
        o.timestamp || new Date(), 0.5 + Math.random() * 0.5,
      ]);

      await this.pool.query(`
        INSERT INTO evidence (id, observation_id, object_id, type, source, timestamp, confidence)
        VALUES ${values}
        ON CONFLICT (id) DO NOTHING
      `, flatValues);

      if (i % 50000 === 0) {
        console.log(`[EXPORT] Evidence: ${Math.min(i + this.batchSize, observations.length)}/${observations.length}`);
      }
    }
  }

  async exportEvents(observations) {
    console.log(`[EXPORT] Exporting events for ${observations.length} observations...`);

    for (let i = 0; i < observations.length; i += this.batchSize) {
      const batch = observations.slice(i, i + this.batchSize);
      const values = batch.map((o, idx) => {
        const offset = idx * 4;
        return `($${offset + 1}, $${offset + 2}, $${offset + 3}, $${offset + 4})`;
      }).join(', ');

      const flatValues = batch.flatMap(o => [
        `evt_${o.id}`, 'observation_created', o.timestamp || new Date(),
        JSON.stringify({ observationId: o.id, objectId: o.objectId }),
      ]);

      await this.pool.query(`
        INSERT INTO events (id, type, timestamp, payload)
        VALUES ${values}
        ON CONFLICT (id) DO NOTHING
      `, flatValues);

      if (i % 50000 === 0) {
        console.log(`[EXPORT] Events: ${Math.min(i + this.batchSize, observations.length)}/${observations.length}`);
      }
    }
  }

  async verify() {
    console.log('[EXPORT] Verifying...');

    const tables = ['zoomers', 'objects', 'observations', 'evidence', 'events'];
    for (const table of tables) {
      const result = await this.pool.query(`SELECT COUNT(*) FROM ${table}`);
      console.log(`[EXPORT] ${table}: ${result.rows[0].count} rows`);
    }
  }

  async close() {
    await this.pool.end();
  }
}

// Exportera
module.exports = PostgresExporter;

// Demo
if (require.main === module) {
  async function runExport() {
    console.log('=== POSTGRES EXPORT ===\n');

    // Generera stad
    const city = new SyntheticCity('Stockholm', 5);
    city.generate();

    // Anslut till PostgreSQL
    const exporter = new PostgresExporter({
      host: 'localhost',
      port: 5433,
      database: 'quixzoom',
      user: 'quixzoom',
      password: 'quixzoom_secure_2026',
    });

    // Exportera
    await exporter.export(city);

    // Stäng
    await exporter.close();

    console.log('\nExport complete!');
  }

  runExport().catch(console.error);
}