""" Reality Signals Engine 10,000+ signals per street → AI-built indexes No fixed KPIs. Flexible. Observation-driven. """ from typing import Dict, List, Optional, Any, Callable from dataclasses import dataclass from datetime import datetime import json @dataclass class RealitySignal: """A single reality signal""" signal_type: str category: str value: float unit: str confidence: float source: str timestamp: str location: Optional[Dict] = None metadata: Optional[Dict] = None def to_dict(self) -> Dict: return { "signal_type": self.signal_type, "category": self.category, "value": round(self.value, 2), "unit": self.unit, "confidence": round(self.confidence, 2), "source": self.source, "timestamp": self.timestamp, "location": self.location, "metadata": self.metadata } class SignalCollector: """Collects reality signals from observations""" def __init__(self): self.signal_definitions = self._load_signal_definitions() def _load_signal_definitions(self) -> Dict: """Define all possible signals""" return { # === VISUAL GEOLOCATION SIGNALS (NEW) === "visual_objects_detected": { "category": "visual", "unit": "count", "indicators": ["street_light", "building", "car", "sign", "tree"] }, "text_detections": { "category": "visual", "unit": "count", "indicators": ["street_names", "business_names", "license_plates"] }, "geometric_features": { "category": "visual", "unit": "count", "indicators": ["vanishing_points", "horizon_lines", "camera_height"] }, "environmental_conditions": { "category": "visual", "unit": "score", "indicators": ["lighting", "weather", "urban_density"] }, "geolocation_confidence": { "category": "visual", "unit": "percentage", "indicators": ["gps_accuracy", "visual_accuracy", "ensemble_confidence"] }, # === 1. HUMAN DAILY LIFE === "walkability": { "category": "human_daily", "unit": "score_0_100", "indicators": ["sidewalk_width", "obstacles", "surface_quality", "shade_coverage"] }, "shade_index": { "category": "human_daily", "unit": "percentage", "indicators": ["tree_coverage", "building_shadow", "awnings", "umbrellas"] }, "heat_stress": { "category": "human_daily", "unit": "celsius_delta", "indicators": ["surface_temperature", "humidity", "wind_speed", "shade_availability"] }, "noise_level": { "category": "human_daily", "unit": "db", "indicators": ["traffic_volume", "construction", "commercial_activity", "crowd_density"] }, "odor_index": { "category": "human_daily", "unit": "score_0_10", "indicators": ["waste_presence", "drainage_condition", "food_vendors", "industrial_nearby"] }, "accessibility": { "category": "human_daily", "unit": "score_0_100", "indicators": ["curb_ramps", "elevator_availability", "door_width", "surface_smoothness"] }, "queue_density": { "category": "human_daily", "unit": "people_per_meter", "indicators": ["queue_length", "service_points", "peak_hours"] }, "wifi_availability": { "category": "human_daily", "unit": "count", "indicators": ["open_networks", "cafe_wifi", "public_wifi", "signal_strength"] }, "mobile_connectivity": { "category": "human_daily", "unit": "score_0_100", "indicators": ["signal_bars", "data_speed", "drop_rate", "carrier_coverage"] }, "public_toilet_index": { "category": "human_daily", "unit": "score_0_100", "indicators": ["availability", "cleanliness", "accessibility", "cost"] }, # === 2. ECONOMIC REALITY === "coffee_price": { "category": "economic", "unit": "usd", "indicators": ["cafe_prices", "street_vendor_prices", "chain_prices"] }, "lunch_price": { "category": "economic", "unit": "usd", "indicators": ["restaurant_prices", "street_food_prices", "market_prices"] }, "beer_price": { "category": "economic", "unit": "usd", "indicators": ["bar_prices", "store_prices", "restaurant_prices"] }, "taxi_cost": { "category": "economic", "unit": "usd_per_km", "indicators": ["metered_rate", "negotiated_rate", "app_rate", "availability"] }, "haircut_price": { "category": "economic", "unit": "usd", "indicators": ["barber_prices", "salon_prices", "street_prices"] }, "laundry_cost": { "category": "economic", "unit": "usd_per_kg", "indicators": ["service_prices", "self_service_prices", "delivery_cost"] }, "street_food_density": { "category": "economic", "unit": "vendors_per_100m", "indicators": ["food_carts", "stalls", "temporary_vendors", "permanent_vendors"] }, "electricity_cost": { "category": "economic", "unit": "usd_per_kwh", "indicators": ["residential_rate", "commercial_rate", "informal_rate"] }, "mobile_data_cost": { "category": "economic", "unit": "usd_per_gb", "indicators": ["prepaid_rate", "postpaid_rate", "tourist_rate"] }, "rent_burden": { "category": "economic", "unit": "percentage_income", "indicators": ["average_rent", "median_income", "informal_rent"] }, "daily_survival_cost": { "category": "economic", "unit": "usd", "indicators": ["food", "transport", "shelter", "utilities", "communication"] }, "hours_per_meal": { "category": "economic", "unit": "hours", "indicators": ["average_wage", "meal_cost", "work_hours"] }, # === 3. IMPLEMENTATION === "reality_gap": { "category": "implementation", "unit": "score_0_100", "indicators": ["planned_vs_observed", "official_vs_actual", "design_vs_function"] }, "infrastructure_reliability": { "category": "implementation", "unit": "score_0_100", "indicators": ["uptime", "failure_rate", "repair_speed", "redundancy"] }, "maintenance_quality": { "category": "implementation", "unit": "score_0_100", "indicators": ["response_time", "repair_quality", "preventive_frequency", "backlog"] }, "public_service_execution": { "category": "implementation", "unit": "score_0_100", "indicators": ["service_availability", "staff_presence", "equipment_function", "queue_efficiency"] }, "road_integrity": { "category": "implementation", "unit": "score_0_100", "indicators": ["surface_condition", "drainage", "markings", "signage"] }, "street_lighting_reliability": { "category": "implementation", "unit": "score_0_100", "indicators": ["coverage", "function_rate", "brightness", "timing"] }, "drainage_performance": { "category": "implementation", "unit": "score_0_100", "indicators": ["capacity", "cleanliness", "flood_frequency", "odor"] }, "garbage_collection_reliability": { "category": "implementation", "unit": "score_0_100", "indicators": ["frequency", "coverage", "segregation", "final_disposal"] }, "construction_completion": { "category": "implementation", "unit": "score_0_100", "indicators": ["planned_vs_actual", "quality", "safety", "timeline_adherence"] }, # === 4. SOCIAL DYNAMICS === "informal_economy": { "category": "social", "unit": "score_0_100", "indicators": ["street_vendors", "unlicensed_services", "cash_transactions", "temporary_stalls"] }, "public_life": { "category": "social", "unit": "score_0_100", "indicators": ["pedestrian_density", "outdoor_seating", "street_performers", "social_gathering"] }, "night_activity": { "category": "social", "unit": "score_0_100", "indicators": ["lighting", "open_businesses", "pedestrian_flow", "safety_perception"] }, "family_presence": { "category": "social", "unit": "score_0_100", "indicators": ["children_visible", "strollers", "play_areas", "schools_nearby"] }, "bicycle_friendliness": { "category": "social", "unit": "score_0_100", "indicators": ["bike_lanes", "parking", "surface", "traffic_safety"] }, "pet_friendliness": { "category": "social", "unit": "score_0_100", "indicators": ["dog_parks", "waste_bins", "water_fountains", "pet_friendly_businesses"] }, "community_activity": { "category": "social", "unit": "score_0_100", "indicators": ["events", "meetings", "sports", "volunteering"] }, "gender_balance": { "category": "social", "unit": "ratio", "indicators": ["observed_flow", "business_ownership", "public_space_usage"] }, "age_diversity": { "category": "social", "unit": "score_0_100", "indicators": ["children", "young_adults", "middle_aged", "elderly"] }, "tourist_pressure": { "category": "social", "unit": "score_0_100", "indicators": ["tourist_density", "souvenir_shops", "hotels", "guided_groups"] }, # === 5. URBAN INTELLIGENCE === "urban_contradiction": { "category": "urban_intelligence", "unit": "score_0_100", "indicators": ["wealth_contrast", "modern_traditional", "planned_informal", "safe_unsafe"] }, "urban_adaptation": { "category": "urban_intelligence", "unit": "score_0_100", "indicators": ["self_built", "improvised", "repurposed", "temporary"] }, "urban_improvisation": { "category": "urban_intelligence", "unit": "score_0_100", "indicators": ["creative_solutions", "material_reuse", "space_optimization", "temporary_fixes"] }, "visual_maintenance": { "category": "urban_intelligence", "unit": "score_0_100", "indicators": ["paint_condition", "cleanliness", "repair_state", "graffiti"] }, "functional_density": { "category": "urban_intelligence", "unit": "score_0_100", "indicators": ["mixed_use", "vertical_integration", "activity_layers", "time_usage"] }, "human_flow_efficiency": { "category": "urban_intelligence", "unit": "score_0_100", "indicators": ["pedestrian_speed", "bottlenecks", "crossing_ease", "wayfinding"] }, "temporary_structure_density": { "category": "urban_intelligence", "unit": "percentage", "indicators": ["temporary_buildings", "construction_sites", "pop_ups", "seasonal"] }, "street_complexity": { "category": "urban_intelligence", "unit": "score_0_100", "indicators": ["intersections", "width_variation", "elevation_change", "obstacles"] }, "vertical_contrast": { "category": "urban_intelligence", "unit": "score_0_100", "indicators": ["height_variation", "skyline_quality", "ground_floor_activity", "upper_floor_usage"] }, "layered_urbanism": { "category": "urban_intelligence", "unit": "score_0_100", "indicators": ["formal_layer", "functional_layer", "informal_layer", "interaction"] } } def collect_from_observation(self, observation: Dict) -> List[RealitySignal]: """Extract signals from a single observation""" signals = [] goid = observation.get("goid", "") findings = observation.get("findings", []) condition = observation.get("overall_condition", 3) location = observation.get("location", {}) # Extract domain from GOID domain = goid.split("-")[0] if "-" in goid else "UNKNOWN" # Generate signals based on observation signals.extend(self._extract_condition_signals(condition, domain, location)) signals.extend(self._extract_finding_signals(findings, domain, location)) signals.extend(self._extract_domain_signals(domain, observation, location)) return signals def _extract_condition_signals( self, condition: int, domain: str, location: Dict ) -> List[RealitySignal]: """Extract signals from condition""" signals = [] # Visual maintenance signal visual_score = (condition - 1) * 25 # 1->0, 5->100 signals.append(RealitySignal( signal_type="visual_maintenance", category="urban_intelligence", value=visual_score, unit="score_0_100", confidence=0.8, source="condition_assessment", timestamp=datetime.utcnow().isoformat(), location=location )) # Infrastructure reliability signal reliability_score = (6 - condition) * 20 # 1->100, 5->20 signals.append(RealitySignal( signal_type="infrastructure_reliability", category="implementation", value=reliability_score, unit="score_0_100", confidence=0.7, source="condition_assessment", timestamp=datetime.utcnow().isoformat(), location=location )) return signals def _extract_finding_signals( self, findings: List[Dict], domain: str, location: Dict ) -> List[RealitySignal]: """Extract signals from findings""" signals = [] for finding in findings: code = finding.get("code", "") # Map finding codes to signals signal_map = { "2100": ("odor_index", 6), # Dirt accumulation "2200": ("visual_maintenance", 4), # Color change "2300": ("road_integrity", 5), # Surface damage "2400": ("urban_contradiction", 3), # Graffiti "4100": ("urban_adaptation", 7), # Missing parts "4200": ("urban_improvisation", 6), # Broken parts "5100": ("human_flow_efficiency", 5), # Blockage "6200": ("drainage_performance", 7), # Water damage } if code in signal_map: signal_type, severity = signal_map[code] signals.append(RealitySignal( signal_type=signal_type, category=self.signal_definitions[signal_type]["category"], value=severity * 10, # Scale to 0-100 unit="score_0_100", confidence=0.75, source=f"finding_{code}", timestamp=datetime.utcnow().isoformat(), location=location, metadata={"finding_code": code} )) return signals def _extract_domain_signals( self, domain: str, observation: Dict, location: Dict ) -> List[RealitySignal]: """Extract domain-specific signals""" signals = [] domain_signals = { "BYG": ["functional_density", "vertical_contrast"], "TRN": ["walkability", "road_integrity", "human_flow_efficiency"], "BEL": ["street_lighting_reliability", "visual_maintenance"], "COM": ["informal_economy", "public_life"], "ENE": ["infrastructure_reliability"] } for signal_type in domain_signals.get(domain, []): signals.append(RealitySignal( signal_type=signal_type, category=self.signal_definitions[signal_type]["category"], value=50.0, # Baseline, will be refined unit="score_0_100", confidence=0.5, source=f"domain_{domain}", timestamp=datetime.utcnow().isoformat(), location=location )) return signals class IndexBuilder: """Builds composite indexes from reality signals""" def __init__(self): self.index_definitions = self._load_index_definitions() def _load_index_definitions(self) -> Dict: """Define composite indexes""" return { # === COMPOSITE INDEXES === "comfort_index": { "signals": ["heat_stress", "noise_level", "shade_index", "odor_index", "accessibility"], "weights": [0.25, 0.20, 0.20, 0.15, 0.20], "category": "human_experience" }, "entrepreneur_index": { "signals": ["street_food_density", "informal_economy", "rent_burden", "public_life", "mobile_connectivity"], "weights": [0.20, 0.25, 0.20, 0.15, 0.20], "category": "economic_opportunity" }, "family_index": { "signals": ["family_presence", "safety_index", "shade_index", "accessibility", "public_toilet_index"], "weights": [0.25, 0.25, 0.15, 0.20, 0.15], "category": "family_friendly" }, "digital_civilization_index": { "signals": ["mobile_connectivity", "wifi_availability", "street_lighting_reliability", "human_flow_efficiency"], "weights": [0.30, 0.25, 0.20, 0.25], "category": "digital" }, "safety_index": { "signals": ["street_lighting_reliability", "road_integrity", "visual_maintenance", "night_activity"], "weights": [0.30, 0.25, 0.20, 0.25], "category": "safety" } } def build_index(self, index_name: str, signals: List[RealitySignal]) -> Optional[Dict]: """ Build a composite index from signals Args: index_name: Name of index to build signals: Available reality signals Returns: Index result """ definition = self.index_definitions.get(index_name) if not definition: return None # Find relevant signals relevant_signals = [] for signal_type, weight in zip(definition["signals"], definition["weights"]): matching = [s for s in signals if s.signal_type == signal_type] if matching: # Average matching signals avg_value = sum(s.value for s in matching) / len(matching) avg_confidence = sum(s.confidence for s in matching) / len(matching) relevant_signals.append({ "signal_type": signal_type, "value": avg_value, "weight": weight, "confidence": avg_confidence }) if not relevant_signals: return None # Calculate weighted score total_weight = sum(s["weight"] for s in relevant_signals) if total_weight == 0: return None weighted_score = sum( s["value"] * s["weight"] for s in relevant_signals ) / total_weight avg_confidence = sum( s["confidence"] * s["weight"] for s in relevant_signals ) / total_weight return { "index_name": index_name, "category": definition["category"], "score": round(weighted_score, 2), "confidence": round(avg_confidence, 2), "signals_used": len(relevant_signals), "total_signals_needed": len(definition["signals"]), "component_signals": relevant_signals, "interpretation": self._interpret_index(index_name, weighted_score) } def build_all_indexes(self, signals: List[RealitySignal]) -> Dict: """Build all possible indexes from signals""" results = {} for index_name in self.index_definitions: result = self.build_index(index_name, signals) if result: results[index_name] = result return results def _interpret_index(self, index_name: str, score: float) -> str: """Generate interpretation for index""" interpretations = { "comfort_index": { 20: "Very uncomfortable", 40: "Uncomfortable", 60: "Moderately comfortable", 80: "Comfortable", 100: "Very comfortable" }, "entrepreneur_index": { 20: "Very difficult to start business", 40: "Difficult", 60: "Moderate", 80: "Easy", 100: "Very easy" }, "family_index": { 20: "Not family-friendly", 40: "Limited family amenities", 60: "Moderately family-friendly", 80: "Family-friendly", 100: "Very family-friendly" }, "digital_civilization_index": { 20: "Poor digital infrastructure", 40: "Limited digital access", 60: "Moderate digital civilization", 80: "Good digital infrastructure", 100: "Excellent digital civilization" } } interp = interpretations.get(index_name, {}) for threshold, text in sorted(interp.items()): if score <= threshold: return text return interp.get(100, "Unknown") class RealitySignalsEngine: """Main engine for reality signals""" def __init__(self): self.collector = SignalCollector() self.index_builder = IndexBuilder() self.signal_store = [] def process_observations(self, observations: List[Dict]) -> Dict: """ Process observations into signals and indexes Args: observations: IOM observations Returns: Complete analysis """ # Collect signals all_signals = [] for obs in observations: signals = self.collector.collect_from_observation(obs) all_signals.extend(signals) self.signal_store.extend(signals) # Build indexes indexes = self.index_builder.build_all_indexes(all_signals) # Calculate statistics signal_count = len(all_signals) signal_types = len(set(s.signal_type for s in all_signals)) categories = len(set(s.category for s in all_signals)) return { "signal_count": signal_count, "signal_types": signal_types, "categories": categories, "signals": [s.to_dict() for s in all_signals], "indexes": indexes, "timestamp": datetime.utcnow().isoformat() } def get_signal_summary(self) -> Dict: """Get summary of all collected signals""" if not self.signal_store: return {"status": "no_signals"} # Group by type by_type = {} for signal in self.signal_store: if signal.signal_type not in by_type: by_type[signal.signal_type] = [] by_type[signal.signal_type].append(signal) # Calculate averages summary = {} for signal_type, signals in by_type.items(): summary[signal_type] = { "count": len(signals), "average_value": round(sum(s.value for s in signals) / len(signals), 2), "average_confidence": round(sum(s.confidence for s in signals) / len(signals), 2), "category": signals[0].category } return { "total_signals": len(self.signal_store), "unique_types": len(by_type), "signal_summary": summary } # Example usage def example_reality_signals(): """Example: Process observations into reality signals""" engine = RealitySignalsEngine() # Example observations from Bangkok observations = [ { "goid": "BYG-FAC-WIN-GLA-001", "overall_condition": 4, "findings": [{"code": "2100", "type": "dirt_accumulation"}], "location": {"lat": 13.7563, "lng": 100.5018} }, { "goid": "TRN-ROD-SUR-001", "overall_condition": 4, "findings": [{"code": "2300", "type": "surface_damage"}], "location": {"lat": 13.7564, "lng": 100.5019} }, { "goid": "COM-DIS-SGN-001", "overall_condition": 3, "findings": [{"code": "2400", "type": "graffiti"}], "location": {"lat": 13.7565, "lng": 100.5020} }, { "goid": "BEL-STR-LIG-001", "overall_condition": 2, "findings": [], "location": {"lat": 13.7566, "lng": 100.5021} } ] # Process result = engine.process_observations(observations) print("=== Reality Signals Analysis ===") print(f"Total signals: {result['signal_count']}") print(f"Signal types: {result['signal_types']}") print(f"Categories: {result['categories']}") print("\nSignals:") for signal in result['signals'][:10]: print(f" {signal['signal_type']}: {signal['value']:.1f} {signal['unit']} (confidence: {signal['confidence']})") print("\nComposite Indexes:") for name, index in result['indexes'].items(): print(f" {name}: {index['score']:.1f} - {index['interpretation']}") # Summary summary = engine.get_signal_summary() print(f"\nTotal signals in store: {summary['total_signals']}") return result if __name__ == '__main__': example_reality_signals()