Bernt
bae705aa97
- Add NFC ePassport roadmap (ICAO 9303, eIDAS) - Add TensorFlow.js edge face detection (BlazeFace) - Add structured audit logger (GDPR-compliant) - Risk scoring support Part of KYC Apple Native UX v1.1.0
561 lines
15 KiB
JavaScript
561 lines
15 KiB
JavaScript
/**
|
|
* QUIXZOOM Urban Foundation Dataset Manager
|
|
*
|
|
* "Git" för hela AI-datasetet.
|
|
* Håller reda på:
|
|
* - alla städer (Reference Cities)
|
|
* - alla objekt
|
|
* - alla etiketter
|
|
* - alla annoteringar
|
|
* - alla modellversioner
|
|
* - alla träningsdataset
|
|
* - licensstatus
|
|
* - datakvalitet
|
|
* - täckningsgrad
|
|
* - versionshistorik
|
|
*/
|
|
|
|
const fs = require('fs');
|
|
const path = require('path');
|
|
const crypto = require('crypto');
|
|
|
|
class UrbanFoundationDatasetManager {
|
|
constructor(config = {}) {
|
|
this.config = {
|
|
dataDir: config.dataDir || './dataset-store',
|
|
...config,
|
|
};
|
|
|
|
// Huvudstrukturer
|
|
this.cities = new Map(); // Reference Cities
|
|
this.objects = new Map(); // Alla objekt
|
|
this.annotations = new Map(); // Alla annoteringar
|
|
this.datasets = new Map(); // Träningsdataset
|
|
this.models = new Map(); // Modellversioner
|
|
this.versions = new Map(); // Versionshistorik
|
|
|
|
// Coverage tracking
|
|
this.coverage = new Map();
|
|
|
|
this.init();
|
|
}
|
|
|
|
init() {
|
|
if (!fs.existsSync(this.config.dataDir)) {
|
|
fs.mkdirSync(this.config.dataDir, { recursive: true });
|
|
}
|
|
this.loadState();
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* REFERENCE CITY
|
|
* ============================================================
|
|
*/
|
|
|
|
addReferenceCity(cityData) {
|
|
const city = {
|
|
id: cityData.id || `city_${crypto.randomBytes(4).toString('hex')}`,
|
|
name: cityData.name,
|
|
country: cityData.country,
|
|
region: cityData.region,
|
|
climate: cityData.climate,
|
|
type: cityData.type, // tropical, mediterranean, nordic, etc.
|
|
location: cityData.location, // { lat, lng }
|
|
timezone: cityData.timezone,
|
|
|
|
// Metadata
|
|
addedAt: new Date().toISOString(),
|
|
status: 'active',
|
|
|
|
// Statistik
|
|
stats: {
|
|
totalObservations: 0,
|
|
totalObjects: 0,
|
|
totalAnnotations: 0,
|
|
lastUpdated: null,
|
|
},
|
|
|
|
// Coverage
|
|
coverage: {},
|
|
};
|
|
|
|
this.cities.set(city.id, city);
|
|
this.saveState();
|
|
|
|
console.log(`[DATASET] Added Reference City: ${city.name} (${city.country})`);
|
|
return city;
|
|
}
|
|
|
|
getReferenceCity(cityId) {
|
|
return this.cities.get(cityId);
|
|
}
|
|
|
|
listReferenceCities() {
|
|
return Array.from(this.cities.values());
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* COVERAGE SCORE
|
|
* ============================================================
|
|
*/
|
|
|
|
updateCoverage(cityId, category, conditions = {}) {
|
|
const city = this.cities.get(cityId);
|
|
if (!city) return null;
|
|
|
|
const coverage = {
|
|
category: category,
|
|
total: conditions.total || 0,
|
|
found: conditions.found || 0,
|
|
percentage: conditions.total > 0 ? (conditions.found / conditions.total * 100).toFixed(1) : 0,
|
|
|
|
// Breakdown
|
|
byTimeOfDay: conditions.byTimeOfDay || {},
|
|
byWeather: conditions.byWeather || {},
|
|
bySeason: conditions.bySeason || {},
|
|
|
|
lastUpdated: new Date().toISOString(),
|
|
};
|
|
|
|
if (!city.coverage[category]) {
|
|
city.coverage[category] = {};
|
|
}
|
|
city.coverage[category] = coverage;
|
|
|
|
this.saveState();
|
|
return coverage;
|
|
}
|
|
|
|
getCoverageReport(cityId) {
|
|
const city = this.cities.get(cityId);
|
|
if (!city) return null;
|
|
|
|
const report = {
|
|
city: city.name,
|
|
overall: 0,
|
|
categories: {},
|
|
gaps: [],
|
|
recommendations: [],
|
|
};
|
|
|
|
let totalPercentage = 0;
|
|
let categoryCount = 0;
|
|
|
|
for (const [category, coverage] of Object.entries(city.coverage)) {
|
|
report.categories[category] = coverage;
|
|
totalPercentage += parseFloat(coverage.percentage);
|
|
categoryCount++;
|
|
|
|
// Identifiera gap
|
|
if (parseFloat(coverage.percentage) < 50) {
|
|
report.gaps.push({
|
|
category,
|
|
coverage: coverage.percentage,
|
|
severity: 'critical',
|
|
});
|
|
} else if (parseFloat(coverage.percentage) < 80) {
|
|
report.gaps.push({
|
|
category,
|
|
coverage: coverage.percentage,
|
|
severity: 'warning',
|
|
});
|
|
}
|
|
|
|
// Generera rekommendationer
|
|
if (coverage.byTimeOfDay) {
|
|
const nightCoverage = coverage.byTimeOfDay.night || 0;
|
|
if (nightCoverage < 20) {
|
|
report.recommendations.push({
|
|
category,
|
|
action: 'collect_night',
|
|
message: `${category}: Only ${nightCoverage}% night coverage. Collect more night observations.`,
|
|
});
|
|
}
|
|
}
|
|
}
|
|
|
|
report.overall = categoryCount > 0 ? (totalPercentage / categoryCount).toFixed(1) : 0;
|
|
|
|
return report;
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* DATA VALUE SCORE
|
|
* ============================================================
|
|
*/
|
|
|
|
calculateDataValueScore(observation) {
|
|
// Information Gain: Hur mycket ny information tillför observationen?
|
|
const informationGain = this.calculateInformationGain(observation);
|
|
|
|
// Coverage Gap: Hur mycket behövs denna kategori?
|
|
const coverageGap = this.calculateCoverageGap(observation);
|
|
|
|
// Image Quality: Teknisk kvalitet
|
|
const imageQuality = observation.quality?.score || 0.5;
|
|
|
|
// Novelty: Hur unik är observationen?
|
|
const novelty = this.calculateNovelty(observation);
|
|
|
|
// Verification Need: Behöver den verifieras?
|
|
const verificationNeed = observation.verified ? 0.3 : 1.0;
|
|
|
|
// Data Value Score = produkt av alla faktorer
|
|
const score =
|
|
informationGain *
|
|
coverageGap *
|
|
imageQuality *
|
|
novelty *
|
|
verificationNeed;
|
|
|
|
return {
|
|
total: Math.round(score * 100),
|
|
breakdown: {
|
|
informationGain: Math.round(informationGain * 100),
|
|
coverageGap: Math.round(coverageGap * 100),
|
|
imageQuality: Math.round(imageQuality * 100),
|
|
novelty: Math.round(novelty * 100),
|
|
verificationNeed: Math.round(verificationNeed * 100),
|
|
},
|
|
};
|
|
}
|
|
|
|
calculateInformationGain(observation) {
|
|
// Om objektet redan har många observationer → lägre gain
|
|
const objectId = observation.objectId;
|
|
if (!objectId) return 1.0;
|
|
|
|
const obj = this.objects.get(objectId);
|
|
if (!obj) return 1.0;
|
|
|
|
const observationCount = obj.observations?.length || 0;
|
|
// Diminishing returns: 1 obs = 1.0, 10 obs = 0.5, 100 obs = 0.1
|
|
return Math.max(0.1, 1 / Math.sqrt(observationCount + 1));
|
|
}
|
|
|
|
calculateCoverageGap(observation) {
|
|
// Om kategorin är underrepresenterad → högre gap
|
|
const cityId = observation.cityId;
|
|
const category = observation.objectType;
|
|
|
|
if (!cityId || !category) return 1.0;
|
|
|
|
const city = this.cities.get(cityId);
|
|
if (!city || !city.coverage[category]) return 1.0;
|
|
|
|
const coverage = parseFloat(city.coverage[category].percentage);
|
|
// 0% coverage = 1.0, 100% coverage = 0.1
|
|
return Math.max(0.1, 1 - (coverage / 100));
|
|
}
|
|
|
|
calculateNovelty(observation) {
|
|
// Hur lik är denna observation tidigare?
|
|
// Simplified: baserat på tid sedan senaste observation av samma typ
|
|
const cityId = observation.cityId;
|
|
const category = observation.objectType;
|
|
|
|
if (!cityId || !category) return 1.0;
|
|
|
|
const recentObs = this.getRecentObservations(cityId, category, 24); // 24h
|
|
if (recentObs.length === 0) return 1.0;
|
|
|
|
// Ju fler nyligen, desto lägre novelty
|
|
return Math.max(0.1, 1 / Math.sqrt(recentObs.length + 1));
|
|
}
|
|
|
|
getRecentObservations(cityId, category, hours) {
|
|
const cutoff = new Date(Date.now() - hours * 3600000);
|
|
const recent = [];
|
|
|
|
for (const obs of this.objects.values()) {
|
|
if (obs.cityId === cityId && obs.type === category) {
|
|
const obsTime = new Date(obs.timestamp);
|
|
if (obsTime > cutoff) {
|
|
recent.push(obs);
|
|
}
|
|
}
|
|
}
|
|
|
|
return recent;
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* DATASET VERSIONING
|
|
* ============================================================
|
|
*/
|
|
|
|
createDatasetVersion(name, config = {}) {
|
|
const version = {
|
|
id: `dataset_${crypto.randomBytes(4).toString('hex')}`,
|
|
name: name,
|
|
createdAt: new Date().toISOString(),
|
|
|
|
// Konfiguration
|
|
cities: config.cities || [],
|
|
categories: config.categories || [],
|
|
conditions: config.conditions || {},
|
|
|
|
// Innehåll
|
|
observations: [],
|
|
annotations: [],
|
|
|
|
// Metadata
|
|
stats: {
|
|
totalObservations: 0,
|
|
totalAnnotations: 0,
|
|
verifiedAnnotations: 0,
|
|
avgQuality: 0,
|
|
},
|
|
|
|
// Spårbarhet
|
|
parentVersion: config.parentVersion || null,
|
|
modelVersion: config.modelVersion || null,
|
|
|
|
// Status
|
|
status: 'draft',
|
|
};
|
|
|
|
this.datasets.set(version.id, version);
|
|
this.saveState();
|
|
|
|
console.log(`[DATASET] Created dataset version: ${name} (${version.id})`);
|
|
return version;
|
|
}
|
|
|
|
addToDataset(datasetId, observationIds, annotationIds) {
|
|
const dataset = this.datasets.get(datasetId);
|
|
if (!dataset) return null;
|
|
|
|
dataset.observations.push(...observationIds);
|
|
dataset.annotations.push(...annotationIds);
|
|
|
|
dataset.stats.totalObservations = dataset.observations.length;
|
|
dataset.stats.totalAnnotations = dataset.annotations.length;
|
|
|
|
this.saveState();
|
|
return dataset;
|
|
}
|
|
|
|
commitDataset(datasetId, message) {
|
|
const dataset = this.datasets.get(datasetId);
|
|
if (!dataset) return null;
|
|
|
|
dataset.status = 'committed';
|
|
dataset.committedAt = new Date().toISOString();
|
|
dataset.commitMessage = message;
|
|
|
|
// Skapa snapshot
|
|
const snapshot = {
|
|
id: `snapshot_${crypto.randomBytes(4).toString('hex')}`,
|
|
datasetId: datasetId,
|
|
timestamp: dataset.committedAt,
|
|
observations: [...dataset.observations],
|
|
annotations: [...dataset.annotations],
|
|
stats: { ...dataset.stats },
|
|
};
|
|
|
|
this.versions.set(snapshot.id, snapshot);
|
|
this.saveState();
|
|
|
|
console.log(`[DATASET] Committed: ${message} (${snapshot.id})`);
|
|
return snapshot;
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* MODEL TRACKING
|
|
* ============================================================
|
|
*/
|
|
|
|
registerModel(modelData) {
|
|
const model = {
|
|
id: `model_${crypto.randomBytes(4).toString('hex')}`,
|
|
name: modelData.name,
|
|
version: modelData.version,
|
|
type: modelData.type, // yolo, grounding_dino, sam2, etc.
|
|
|
|
// Träningsinfo
|
|
datasetId: modelData.datasetId,
|
|
trainingConfig: modelData.trainingConfig,
|
|
|
|
// Prestanda
|
|
metrics: modelData.metrics || {},
|
|
|
|
// Status
|
|
status: 'trained',
|
|
createdAt: new Date().toISOString(),
|
|
};
|
|
|
|
this.models.set(model.id, model);
|
|
this.saveState();
|
|
|
|
console.log(`[DATASET] Registered model: ${model.name} v${model.version}`);
|
|
return model;
|
|
}
|
|
|
|
getModelLineage(modelId) {
|
|
const model = this.models.get(modelId);
|
|
if (!model) return null;
|
|
|
|
const lineage = [model];
|
|
|
|
// Hitta föräldradataset
|
|
let datasetId = model.datasetId;
|
|
while (datasetId) {
|
|
const dataset = this.datasets.get(datasetId);
|
|
if (!dataset) break;
|
|
|
|
lineage.push({
|
|
type: 'dataset',
|
|
id: dataset.id,
|
|
name: dataset.name,
|
|
});
|
|
|
|
datasetId = dataset.parentVersion;
|
|
}
|
|
|
|
return lineage;
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* PERSISTENCE
|
|
* ============================================================
|
|
*/
|
|
|
|
saveState() {
|
|
const state = {
|
|
cities: Array.from(this.cities.entries()),
|
|
objects: Array.from(this.objects.entries()),
|
|
annotations: Array.from(this.annotations.entries()),
|
|
datasets: Array.from(this.datasets.entries()),
|
|
models: Array.from(this.models.entries()),
|
|
versions: Array.from(this.versions.entries()),
|
|
};
|
|
|
|
const statePath = path.join(this.config.dataDir, 'dataset-state.json');
|
|
fs.writeFileSync(statePath, JSON.stringify(state, null, 2));
|
|
}
|
|
|
|
loadState() {
|
|
const statePath = path.join(this.config.dataDir, 'dataset-state.json');
|
|
if (!fs.existsSync(statePath)) return;
|
|
|
|
try {
|
|
const state = JSON.parse(fs.readFileSync(statePath));
|
|
|
|
if (state.cities) this.cities = new Map(state.cities);
|
|
if (state.objects) this.objects = new Map(state.objects);
|
|
if (state.annotations) this.annotations = new Map(state.annotations);
|
|
if (state.datasets) this.datasets = new Map(state.datasets);
|
|
if (state.models) this.models = new Map(state.models);
|
|
if (state.versions) this.versions = new Map(state.versions);
|
|
} catch (error) {
|
|
console.warn('[DATASET] Failed to load state:', error.message);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* ============================================================
|
|
* RAPPORTER
|
|
* ============================================================
|
|
*/
|
|
|
|
generateReport() {
|
|
const report = {
|
|
generatedAt: new Date().toISOString(),
|
|
summary: {
|
|
totalCities: this.cities.size,
|
|
totalObjects: this.objects.size,
|
|
totalAnnotations: this.annotations.size,
|
|
totalDatasets: this.datasets.size,
|
|
totalModels: this.models.size,
|
|
},
|
|
cities: [],
|
|
datasets: [],
|
|
models: [],
|
|
};
|
|
|
|
for (const city of this.cities.values()) {
|
|
const cityReport = this.getCoverageReport(city.id);
|
|
report.cities.push(cityReport);
|
|
}
|
|
|
|
for (const dataset of this.datasets.values()) {
|
|
report.datasets.push({
|
|
id: dataset.id,
|
|
name: dataset.name,
|
|
status: dataset.status,
|
|
stats: dataset.stats,
|
|
});
|
|
}
|
|
|
|
for (const model of this.models.values()) {
|
|
report.models.push({
|
|
id: model.id,
|
|
name: model.name,
|
|
version: model.version,
|
|
metrics: model.metrics,
|
|
});
|
|
}
|
|
|
|
return report;
|
|
}
|
|
}
|
|
|
|
module.exports = UrbanFoundationDatasetManager;
|
|
|
|
// Demo
|
|
if (require.main === module) {
|
|
const manager = new UrbanFoundationDatasetManager();
|
|
|
|
// Lägg till Reference Cities
|
|
const bangkok = manager.addReferenceCity({
|
|
name: 'Bangkok',
|
|
country: 'TH',
|
|
region: 'Southeast Asia',
|
|
climate: 'tropical',
|
|
type: 'tropical_megacity',
|
|
location: { lat: 13.7563, lng: 100.5018 },
|
|
timezone: 'Asia/Bangkok',
|
|
});
|
|
|
|
const torrevieja = manager.addReferenceCity({
|
|
name: 'Torrevieja',
|
|
country: 'ES',
|
|
region: 'Mediterranean',
|
|
climate: 'mediterranean',
|
|
type: 'mediterranean_coastal',
|
|
location: { lat: 37.9838, lng: -0.6828 },
|
|
timezone: 'Europe/Madrid',
|
|
});
|
|
|
|
// Uppdatera coverage
|
|
manager.updateCoverage(bangkok.id, 'street_lamp', {
|
|
total: 1000,
|
|
found: 940,
|
|
byTimeOfDay: { day: 500, evening: 300, night: 140 },
|
|
byWeather: { clear: 600, cloudy: 250, rain: 90 },
|
|
});
|
|
|
|
manager.updateCoverage(bangkok.id, 'traffic_sign', {
|
|
total: 500,
|
|
found: 455,
|
|
byTimeOfDay: { day: 300, evening: 120, night: 35 },
|
|
});
|
|
|
|
// Generera coverage report
|
|
const report = manager.getCoverageReport(bangkok.id);
|
|
console.log('\n=== BANGKOK COVERAGE REPORT ===');
|
|
console.log(JSON.stringify(report, null, 2));
|
|
|
|
// Generera full rapport
|
|
const fullReport = manager.generateReport();
|
|
console.log('\n=== FULL REPORT ===');
|
|
console.log(JSON.stringify(fullReport.summary, null, 2));
|
|
}
|