/** * QUIXZOOM Capture Pipeline — AI Preprocessing Worker * * Köar: quixzoom:processing:queue * Utför: * 1. Blur detection * 2. Duplicate detection * 3. OCR * 4. Object detection * 5. Scene classification * 6. Quality scoring * 7. Thumbnail generation * * Teknik: Node.js, TensorFlow.js, Sharp, Tesseract.js */ const { S3Client, GetObjectCommand, PutObjectCommand } = require('@aws-sdk/client-s3'); const sharp = require('sharp'); const tf = require('@tensorflow/tfjs-node'); const { createWorker } = require('tesseract.js'); const crypto = require('crypto'); // R2 klient const r2Client = new S3Client({ region: 'auto', endpoint: process.env.R2_ENDPOINT, credentials: { accessKeyId: process.env.R2_ACCESS_KEY_ID, secretAccessKey: process.env.R2_SECRET_ACCESS_KEY, }, }); const BUCKET_NAME = process.env.R2_BUCKET_NAME || 'quixzoom-capture'; // AI-modeller (laddas vid start) let objectDetectionModel = null; let sceneClassificationModel = null; let blurDetectionModel = null; /** * Initiera AI-modeller */ async function initializeModels() { console.log('[AI] Initializing models...'); // Ladda objektdetekteringsmodell (COCO-SSD) objectDetectionModel = await tf.loadGraphModel('file://./models/coco-ssd/model.json'); console.log('[AI] Object detection model loaded'); // Ladda scenklassificeringsmodell sceneClassificationModel = await tf.loadGraphModel('file://./models/scene-classification/model.json'); console.log('[AI] Scene classification model loaded'); // Ladda blur-detekteringsmodell blurDetectionModel = await tf.loadGraphModel('file://./models/blur-detection/model.json'); console.log('[AI] Blur detection model loaded'); console.log('[AI] All models initialized'); } /** * Hämta bild från R2 */ async function getImageFromR2(fileName) { const response = await r2Client.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: `originals/${fileName}`, })); const chunks = []; for await (const chunk of response.Body) { chunks.push(chunk); } return Buffer.concat(chunks); } /** * Hämta metadata från R2 */ async function getMetadataFromR2(captureId) { const datePath = captureId.substring(0, 4) + '/' + captureId.substring(4, 6) + '/' + captureId.substring(6, 8); const metadataFileName = `${datePath}/${captureId}.json`; const response = await r2Client.send(new GetObjectCommand({ Bucket: BUCKET_NAME, Key: `metadata/${metadataFileName}`, })); const chunks = []; for await (const chunk of response.Body) { chunks.push(chunk); } return JSON.parse(Buffer.concat(chunks).toString()); } /** * Spara metadata till R2 */ async function saveMetadataToR2(captureId, metadata) { const datePath = captureId.substring(0, 4) + '/' + captureId.substring(4, 6) + '/' + captureId.substring(6, 8); const metadataFileName = `${datePath}/${captureId}.json`; await r2Client.send(new PutObjectCommand({ Bucket: BUCKET_NAME, Key: `metadata/${metadataFileName}`, Body: JSON.stringify(metadata, null, 2), ContentType: 'application/json', })); } /** * 1. Blur Detection * Beräknar Laplacian variance för att detektera oskärpa */ async function detectBlur(imageBuffer) { const { data, info } = await sharp(imageBuffer) .greyscale() .raw() .toBuffer({ resolveWithObject: true }); // Beräkna Laplacian variance const width = info.width; const height = info.height; let sum = 0; let sumSq = 0; for (let y = 1; y < height - 1; y++) { for (let x = 1; x < width - 1; x++) { const idx = y * width + x; const laplacian = -4 * data[idx] + data[idx - 1] + data[idx + 1] + data[idx - width] + data[idx + width]; sum += laplacian; sumSq += laplacian * laplacian; } } const mean = sum / ((width - 2) * (height - 2)); const variance = (sumSq / ((width - 2) * (height - 2))) - (mean * mean); // Normalisera till 0-100 const blurScore = Math.min(100, Math.max(0, variance / 100)); return { score: blurScore, variance: variance, isBlurred: variance < 100, // Tröskelvärde }; } /** * 2. Duplicate Detection * Jämför perceptuella hashar */ async function detectDuplicates(imageBuffer, captureId) { // Generera perceptuell hash (pHash) const hash = await generatePerceptualHash(imageBuffer); // TODO: Sök i databas efter liknande hashar // För nu: returnera tom lista return { hash: hash, duplicates: [], similarityThreshold: 0.9, }; } /** * Generera perceptuell hash */ async function generatePerceptualHash(imageBuffer) { const resized = await sharp(imageBuffer) .resize(32, 32, { fit: 'fill' }) .greyscale() .raw() .toBuffer(); // Beräkna DCT (förenklad version) const dct = computeDCT(resized); // Generera hash från DCT-koefficienter const hash = crypto.createHash('md5').update(dct).digest('hex'); return hash; } /** * Förenklad DCT-beräkning */ function computeDCT(data) { // Förenklad implementation — använd bibliotek i produktion const mean = data.reduce((a, b) => a + b, 0) / data.length; const binary = data.map(pixel => pixel > mean ? '1' : '0').join(''); return Buffer.from(binary, 'binary'); } /** * 3. OCR (Optical Character Recognition) * Extraherar text från bilder */ async function performOCR(imageBuffer) { const worker = await createWorker('eng+tha'); // Konvertera till format som Tesseract accepterar const { data } = await sharp(imageBuffer) .png() .toBuffer(); const result = await worker.recognize(data); await worker.terminate(); return { text: result.data.text, confidence: result.data.confidence, words: result.data.words.map(w => ({ text: w.text, confidence: w.confidence, bbox: w.bbox, })), }; } /** * 4. Object Detection * Detekterar objekt med COCO-SSD */ async function detectObjects(imageBuffer) { // Konvertera till Tensor const tensor = await sharpToTensor(imageBuffer); // Kör inferens const predictions = await objectDetectionModel.detect(tensor); // Frigör minne tensor.dispose(); // Filtrera och formatera resultat const objects = predictions.map(p => ({ class: p.class, score: p.score, bbox: p.bbox, // [x, y, width, height] })).filter(p => p.score > 0.5); // Tröskelvärde // Gruppera efter klass const grouped = {}; objects.forEach(obj => { if (!grouped[obj.class]) grouped[obj.class] = []; grouped[obj.class].push(obj); }); return { objects: objects, grouped: grouped, totalCount: objects.length, }; } /** * 5. Scene Classification * Klassificerar scenen (väg, byggnad, natur, etc.) */ async function classifyScene(imageBuffer) { const tensor = await sharpToTensor(imageBuffer, [224, 224]); const predictions = await sceneClassificationModel.predict(tensor).data(); tensor.dispose(); // Mappa till etiketter const labels = [ 'road', 'highway', 'intersection', 'building', 'residential', 'commercial', 'industrial', 'park', 'water', 'bridge', 'tunnel', 'construction', 'parking', 'sidewalk', 'alley' ]; const results = predictions .map((score, idx) => ({ label: labels[idx] || 'unknown', score })) .sort((a, b) => b.score - a.score) .slice(0, 5); return { topScene: results[0], allScenes: results, }; } /** * 6. Quality Scoring * Kombinerad kvalitetsbedömning */ async function calculateQualityScore(blurResult, objectResult, ocrResult) { const blurWeight = 0.4; const objectWeight = 0.3; const ocrWeight = 0.2; const resolutionWeight = 0.1; const blurScore = blurResult.score; const objectScore = Math.min(100, objectResult.totalCount * 10); const ocrScore = ocrResult.confidence || 0; const qualityScore = blurScore * blurWeight + objectScore * objectWeight + ocrScore * ocrWeight + 100 * resolutionWeight; // Anta full upplösning return { overall: Math.round(qualityScore), components: { blur: blurScore, objects: objectScore, ocr: ocrScore, resolution: 100, }, isUsable: qualityScore > 60, }; } /** * 7. Thumbnail Generation * Generera thumbnails i olika storlekar */ async function generateThumbnails(imageBuffer, captureId) { const sizes = [ { name: 'thumbnail', width: 200, height: 200 }, { name: 'preview', width: 800, height: 600 }, { name: 'medium', width: 1600, height: 1200 }, ]; const thumbnails = {}; for (const size of sizes) { const resized = await sharp(imageBuffer) .resize(size.width, size.height, { fit: 'inside', withoutEnlargement: true }) .jpeg({ quality: 85 }) .toBuffer(); const fileName = `thumbnails/${size.name}/${captureId}.jpg`; await r2Client.send(new PutObjectCommand({ Bucket: BUCKET_NAME, Key: fileName, Body: resized, ContentType: 'image/jpeg', })); thumbnails[size.name] = `https://${BUCKET_NAME}.r2.cloudflarestorage.com/${fileName}`; } return thumbnails; } /** * Konvertera Sharp buffer till TensorFlow tensor */ async function sharpToTensor(imageBuffer, size = [640, 640]) { const { data, info } = await sharp(imageBuffer) .resize(size[0], size[1], { fit: 'fill' }) .raw() .toBuffer({ resolveWithObject: true }); return tf.tidy(() => { const image = tf.tensor3d(new Uint8Array(data), [info.height, info.width, 3]); return image.expandDims(0).toFloat().div(255.0); }); } /** * Huvudbearbetningsfunktion */ async function processImage(captureId, fileName) { console.log(`[PROCESS] Starting processing for ${captureId}`); try { // Hämta bild och metadata const imageBuffer = await getImageFromR2(fileName); const metadata = await getMetadataFromR2(captureId); // Uppdatera status metadata.aiAnalysis.status = 'processing'; metadata.processedAt = new Date().toISOString(); // 1. Blur Detection console.log(`[PROCESS] ${captureId} — Blur detection`); const blurResult = await detectBlur(imageBuffer); metadata.aiAnalysis.blurScore = blurResult; // 2. Duplicate Detection console.log(`[PROCESS] ${captureId} — Duplicate detection`); const duplicateResult = await detectDuplicates(imageBuffer, captureId); metadata.aiAnalysis.duplicates = duplicateResult; // 3. OCR console.log(`[PROCESS] ${captureId} — OCR`); const ocrResult = await performOCR(imageBuffer); metadata.aiAnalysis.ocrText = ocrResult; // 4. Object Detection console.log(`[PROCESS] ${captureId} — Object detection`); const objectResult = await detectObjects(imageBuffer); metadata.aiAnalysis.objectsDetected = objectResult; // 5. Scene Classification console.log(`[PROCESS] ${captureId} — Scene classification`); const sceneResult = await classifyScene(imageBuffer); metadata.aiAnalysis.sceneClassification = sceneResult; // 6. Quality Scoring console.log(`[PROCESS] ${captureId} — Quality scoring`); const qualityResult = await calculateQualityScore(blurResult, objectResult, ocrResult); metadata.aiAnalysis.qualityScore = qualityResult; // 7. Thumbnail Generation console.log(`[PROCESS] ${captureId} — Thumbnail generation`); const thumbnails = await generateThumbnails(imageBuffer, captureId); metadata.file.thumbnails = thumbnails; // Uppdatera status metadata.aiAnalysis.status = 'completed'; // Spara uppdaterad metadata await saveMetadataToR2(captureId, metadata); console.log(`[PROCESS] ${captureId} — Completed`); return { captureId, status: 'completed', qualityScore: qualityResult.overall, isUsable: qualityResult.isUsable, }; } catch (error) { console.error(`[PROCESS] ${captureId} — Error:`, error); // Uppdatera metadata med fel const metadata = await getMetadataFromR2(captureId); metadata.aiAnalysis.status = 'failed'; metadata.aiAnalysis.error = error.message; await saveMetadataToR2(captureId, metadata); throw error; } } /** * Worker-loop — lyssna på kön */ async function startWorker() { console.log('[WORKER] Starting AI preprocessing worker...'); // Initiera modeller await initializeModels(); // TODO: Implementera faktisk kö-lyssning // För nu: pollning eller Redis Pub/Sub console.log('[WORKER] Ready to process images'); // Exempel på manuell bearbetning // await processImage('capture-id', 'path/to/file.jpg'); } // Starta worker om filen körs direkt if (require.main === module) { startWorker().catch(console.error); } module.exports = { processImage, startWorker, detectBlur, detectDuplicates, performOCR, detectObjects, classifyScene, calculateQualityScore, generateThumbnails, };