""" quiXzoom Data Ingestion Pipeline Collect and process images from quiXzoom app for AI training """ from typing import Dict, List, Optional from dataclasses import dataclass from pathlib import Path import json import os @dataclass class ZoomerSubmission: """A photo submission from a Zoomer""" submission_id: str zoomer_id: str image_url: str location: Dict # lat, lng timestamp: str mission_type: str tags: List[str] quality_score: float # 0-1 class QuixzoomDataIngestion: """ Ingest data from quiXzoom app for AI training Flow: 1. Fetch submissions from quiXzoom API 2. Filter by quality (blur, lighting, composition) 3. Auto-annotate with existing AI model 4. Human review queue 5. Export to training format """ def __init__(self, output_path: str = "/tmp/quixzoom_training"): self.output_path = Path(output_path) self.submissions: List[ZoomerSubmission] = [] # Create directories (self.output_path / "images").mkdir(parents=True, exist_ok=True) (self.output_path / "labels").mkdir(parents=True, exist_ok=True) (self.output_path / "review_queue").mkdir(parents=True, exist_ok=True) def fetch_submissions( self, api_endpoint: str = "https://api.quixzoom.com/v1/submissions", api_key: Optional[str] = None, limit: int = 1000 ) -> List[ZoomerSubmission]: """Fetch submissions from quiXzoom API""" print(f"Fetching up to {limit} submissions from {api_endpoint}...") # In production, make actual API call # For now, return placeholder submissions = [] for i in range(min(limit, 100)): # Demo: 100 submissions submission = ZoomerSubmission( submission_id=f"SUB_{i:06d}", zoomer_id=f"ZOOMER_{i % 50:04d}", image_url=f"https://cdn.quixzoom.com/submissions/SUB_{i:06d}.jpg", location={"lat": 59.3293 + (i * 0.001), "lng": 18.0686 + (i * 0.001)}, timestamp="2026-06-26T12:00:00Z", mission_type="infrastructure_check", tags=["street_light", "sidewalk", "road"], quality_score=0.7 + (i % 30) / 100 ) submissions.append(submission) self.submissions = submissions print(f"Fetched {len(submissions)} submissions") return submissions def filter_quality( self, min_quality: float = 0.6, require_gps: bool = True, require_daylight: bool = True ) -> List[ZoomerSubmission]: """Filter submissions by quality criteria""" print(f"\nFiltering {len(self.submissions)} submissions...") filtered = [] for sub in self.submissions: # Quality score if sub.quality_score < min_quality: continue # GPS required if require_gps and not sub.location: continue # Daylight (simplified check) if require_daylight: # In production, check timestamp + location for daylight pass filtered.append(sub) print(f"Filtered: {len(filtered)}/{len(self.submissions)} passed quality checks") return filtered def auto_annotate(self, submissions: List[ZoomerSubmission]) -> Dict: """Auto-annotate images with existing AI model""" print(f"\nAuto-annotating {len(submissions)} images...") import sys sys.path.insert(0, '/home/bernt/.openclaw/workspace/iom') from ai_pipeline.real_ai import RealAIClassifier classifier = RealAIClassifier(use_real_ai=True) annotations = {} for sub in submissions[:10]: # Demo: annotate first 10 # In production, download image and run inference # For now, simulate annotations[sub.submission_id] = { "objects": [ {"class": "street_light", "confidence": 0.85, "bbox": [100, 200, 50, 150]}, {"class": "sidewalk", "confidence": 0.92, "bbox": [0, 400, 640, 80]} ], "scene_type": "street_view", "quality": sub.quality_score } print(f"Auto-annotated {len(annotations)} images") return annotations def create_review_queue(self, annotations: Dict): """Create human review queue for low-confidence annotations""" print(f"\nCreating review queue...") review_items = [] for sub_id, annotation in annotations.items(): # Flag low-confidence detections for review for obj in annotation["objects"]: if obj["confidence"] < 0.8: review_items.append({ "submission_id": sub_id, "object": obj, "reason": "low_confidence" }) # Save review queue review_path = self.output_path / "review_queue" / "pending.json" with open(review_path, "w") as f: json.dump(review_items, f, indent=2) print(f"Review queue: {len(review_items)} items flagged") return review_items def export_to_training_format( self, annotations: Dict, split_ratios: Dict = {"train": 0.7, "val": 0.2, "test": 0.1} ): """Export annotations to YOLO training format""" print(f"\nExporting to training format...") import random # Split data items = list(annotations.items()) random.shuffle(items) n = len(items) train_end = int(n * split_ratios["train"]) val_end = train_end + int(n * split_ratios["val"]) splits = { "train": items[:train_end], "val": items[train_end:val_end], "test": items[val_end:] } # Export each split for split_name, split_items in splits.items(): split_dir = self.output_path / split_name split_dir.mkdir(exist_ok=True) for sub_id, annotation in split_items: # Save label file label_path = split_dir / f"{sub_id}.txt" with open(label_path, "w") as f: for obj in annotation["objects"]: # YOLO format: class x_center y_center width height # In production, convert bbox to normalized coordinates f.write(f"0 0.5 0.5 0.1 0.1\n") # Placeholder # Create data.yaml data_yaml = { "path": str(self.output_path.absolute()), "train": "train", "val": "val", "test": "test", "nc": 10, "names": [ "street_light", "traffic_sign", "bench", "trash_can", "sidewalk", "road", "building", "bridge", "tree", "graffiti" ] } with open(self.output_path / "data.yaml", "w") as f: json.dump(data_yaml, f, indent=2) print(f"Exported:") for split_name, split_items in splits.items(): print(f" {split_name}: {len(split_items)} images") def run_full_pipeline(self) -> Dict: """Run complete data ingestion pipeline""" print("=" * 60) print("QUIXZOOM DATA INGESTION") print("=" * 60) # 1. Fetch submissions submissions = self.fetch_submissions() # 2. Filter quality filtered = self.filter_quality() # 3. Auto-annotate annotations = self.auto_annotate(filtered) # 4. Create review queue review_items = self.create_review_queue(annotations) # 5. Export self.export_to_training_format(annotations) print("\n" + "=" * 60) print("INGESTION COMPLETE") print("=" * 60) return { "total_submissions": len(submissions), "filtered_submissions": len(filtered), "annotated": len(annotations), "review_queue": len(review_items) } # Example usage def example_ingestion(): """Example: Data ingestion""" ingestion = QuixzoomDataIngestion() results = ingestion.run_full_pipeline() print("\nResults:") for key, value in results.items(): print(f" {key}: {value}") return results if __name__ == '__main__': example_ingestion()