""" Semantic Graph — Connects Reality Signals into a knowledge graph Enables AI to discover relationships and build custom indexes """ from typing import Dict, List, Optional, Set, Tuple from dataclasses import dataclass from datetime import datetime import json @dataclass class SignalNode: """A node in the semantic graph""" signal_id: str signal_type: str category: str value: float unit: str location: Optional[Dict] timestamp: str relationships: List[str] # IDs of related signals def to_dict(self) -> Dict: return { "signal_id": self.signal_id, "signal_type": self.signal_type, "category": self.category, "value": round(self.value, 2), "unit": self.unit, "location": self.location, "timestamp": self.timestamp, "relationships": self.relationships } class SemanticGraph: """Knowledge graph for reality signals""" def __init__(self): self.nodes: Dict[str, SignalNode] = {} self.edges: List[Dict] = [] self.signal_type_index: Dict[str, List[str]] = {} # type -> signal_ids self.location_index: Dict[str, List[str]] = {} # location_hash -> signal_ids def add_signal(self, signal: Dict) -> str: """Add a signal to the graph""" signal_id = f"{signal['signal_type']}_{signal.get('timestamp', datetime.utcnow().isoformat())}" node = SignalNode( signal_id=signal_id, signal_type=signal['signal_type'], category=signal['category'], value=signal['value'], unit=signal['unit'], location=signal.get('location'), timestamp=signal.get('timestamp', datetime.utcnow().isoformat()), relationships=[] ) self.nodes[signal_id] = node # Index by type if node.signal_type not in self.signal_type_index: self.signal_type_index[node.signal_type] = [] self.signal_type_index[node.signal_type].append(signal_id) # Index by location if node.location: loc_hash = self._hash_location(node.location) if loc_hash not in self.location_index: self.location_index[loc_hash] = [] self.location_index[loc_hash].append(signal_id) return signal_id def add_relationship(self, source_id: str, target_id: str, relation_type: str, strength: float): """Add a relationship between two signals""" if source_id in self.nodes and target_id in self.nodes: self.edges.append({ "source": source_id, "target": target_id, "type": relation_type, "strength": round(strength, 2) }) # Update node relationships self.nodes[source_id].relationships.append(target_id) self.nodes[target_id].relationships.append(source_id) def find_related(self, signal_id: str, relation_type: Optional[str] = None) -> List[Dict]: """Find signals related to a given signal""" if signal_id not in self.nodes: return [] related = [] for edge in self.edges: if edge["source"] == signal_id or edge["target"] == signal_id: if relation_type is None or edge["type"] == relation_type: other_id = edge["target"] if edge["source"] == signal_id else edge["source"] if other_id in self.nodes: related.append({ "signal": self.nodes[other_id].to_dict(), "relationship": edge["type"], "strength": edge["strength"] }) return related def find_by_type(self, signal_type: str) -> List[Dict]: """Find all signals of a given type""" signal_ids = self.signal_type_index.get(signal_type, []) return [self.nodes[sid].to_dict() for sid in signal_ids if sid in self.nodes] def find_by_location(self, location: Dict, radius_meters: float = 100) -> List[Dict]: """Find signals near a location""" # Simplified: exact location hash match # In production: use PostGIS spatial query loc_hash = self._hash_location(location) signal_ids = self.location_index.get(loc_hash, []) return [self.nodes[sid].to_dict() for sid in signal_ids if sid in self.nodes] def get_subgraph(self, signal_types: List[str]) -> Dict: """Get a subgraph containing only specified signal types""" nodes = {} edges = [] for signal_type in signal_types: for signal_id in self.signal_type_index.get(signal_type, []): if signal_id in self.nodes: nodes[signal_id] = self.nodes[signal_id].to_dict() # Find edges between these nodes node_ids = set(nodes.keys()) for edge in self.edges: if edge["source"] in node_ids and edge["target"] in node_ids: edges.append(edge) return { "nodes": list(nodes.values()), "edges": edges, "node_count": len(nodes), "edge_count": len(edges) } def discover_correlations(self) -> List[Dict]: """Discover correlations between signal types""" correlations = [] # Get all signal types types = list(self.signal_type_index.keys()) # Check correlations between pairs for i, type_a in enumerate(types): for type_b in types[i+1:]: signals_a = [self.nodes[sid] for sid in self.signal_type_index.get(type_a, []) if sid in self.nodes] signals_b = [self.nodes[sid] for sid in self.signal_type_index.get(type_b, []) if sid in self.nodes] if signals_a and signals_b: # Simple correlation: compare averages avg_a = sum(s.value for s in signals_a) / len(signals_a) avg_b = sum(s.value for s in signals_b) / len(signals_b) # Correlation strength (simplified) correlation = 1 - abs(avg_a - avg_b) / max(avg_a, avg_b, 1) if correlation > 0.5: # Threshold correlations.append({ "type_a": type_a, "type_b": type_b, "correlation": round(correlation, 2), "avg_a": round(avg_a, 2), "avg_b": round(avg_b, 2), "sample_size": min(len(signals_a), len(signals_b)) }) # Sort by correlation strength correlations.sort(key=lambda x: x["correlation"], reverse=True) return correlations def _hash_location(self, location: Dict) -> str: """Hash location for indexing""" lat = location.get("lat", location.get("latitude", 0)) lng = location.get("lng", location.get("longitude", 0)) # Round to ~100m precision return f"{round(lat, 3)}_{round(lng, 3)}" def get_stats(self) -> Dict: """Get graph statistics""" return { "total_nodes": len(self.nodes), "total_edges": len(self.edges), "signal_types": len(self.signal_type_index), "locations": len(self.location_index), "type_distribution": { stype: len(sids) for stype, sids in self.signal_type_index.items() } } class AIReasoningLayer: """AI reasoning layer for dynamic index creation""" def __init__(self, semantic_graph: SemanticGraph): self.graph = semantic_graph self.index_templates = self._load_index_templates() def _load_index_templates(self) -> Dict: """Load index templates for common queries""" return { "family_friendly": { "signals": ["traffic_intensity", "sidewalk_width", "noise_level", "air_quality", "playground", "greenery", "schools", "lighting", "crosswalks", "accessibility"], "weights": [0.15, 0.10, 0.10, 0.10, 0.15, 0.10, 0.10, 0.10, 0.05, 0.05], "description": "Measures attractiveness for families with children" }, "restaurant_success": { "signals": ["pedestrian_flow", "evening_lighting", "shade_coverage", "public_transit", "dwell_time", "competition_density", "rent_burden", "parking"], "weights": [0.20, 0.15, 0.10, 0.15, 0.15, 0.10, 0.10, 0.05], "description": "Predicts commercial success for restaurants" }, "investment_readiness": { "signals": ["infrastructure_reliability", "maintenance_quality", "urban_contradiction", "informal_economy", "mobile_connectivity", "road_integrity", "street_lighting_reliability"], "weights": [0.20, 0.15, 0.15, 0.10, 0.15, 0.15, 0.10], "description": "Measures readiness for infrastructure investment" }, "tourism_potential": { "signals": ["public_life", "night_activity", "walkability", "shade_index", "public_toilet_index", "wifi_availability", "safety_index", "visual_maintenance"], "weights": [0.15, 0.15, 0.15, 0.10, 0.10, 0.10, 0.15, 0.10], "description": "Measures tourism attractiveness" }, "climate_resilience": { "signals": ["heat_stress", "shade_index", "drainage_performance", "tree_coverage", "surface_temperature", "green_space", "water_retention"], "weights": [0.20, 0.15, 0.20, 0.15, 0.15, 0.10, 0.05], "description": "Measures urban climate resilience" } } def answer_query(self, query: str) -> Dict: """ Answer a natural language query by building a custom index Examples: - "How attractive is this area for families?" - "What signals correlate with restaurant success?" - "Show me a climate resilience index" """ # Parse query query_lower = query.lower() # Match to template matched_template = None for template_name, template in self.index_templates.items(): if any(keyword in query_lower for keyword in template_name.split("_")): matched_template = template_name break if not matched_template: # Try to infer from query return self._infer_index_from_query(query) # Build index from template template = self.index_templates[matched_template] return self._build_index_from_template(matched_template, template) def _build_index_from_template(self, name: str, template: Dict) -> Dict: """Build an index from a template""" signals_needed = template["signals"] weights = template["weights"] # Find available signals available_signals = [] missing_signals = [] for signal_type, weight in zip(signals_needed, weights): signals = self.graph.find_by_type(signal_type) if signals: # Average value avg_value = sum(s["value"] for s in signals) / len(signals) avg_confidence = sum(s.get("confidence", 0.5) for s in signals) / len(signals) available_signals.append({ "signal_type": signal_type, "value": avg_value, "weight": weight, "confidence": avg_confidence, "sample_size": len(signals) }) else: missing_signals.append(signal_type) if not available_signals: return { "status": "insufficient_data", "index_name": name, "missing_signals": missing_signals, "message": "No required signals available in the graph" } # Calculate weighted score total_weight = sum(s["weight"] for s in available_signals) if total_weight == 0: total_weight = 1 score = sum(s["value"] * s["weight"] for s in available_signals) / total_weight confidence = sum(s["confidence"] * s["weight"] for s in available_signals) / total_weight return { "status": "success", "index_name": name, "score": round(score, 2), "confidence": round(confidence, 2), "signals_used": len(available_signals), "signals_needed": len(signals_needed), "missing_signals": missing_signals, "component_signals": available_signals, "description": template["description"], "interpretation": self._interpret_score(name, score) } def _infer_index_from_query(self, query: str) -> Dict: """Infer index requirements from query""" # Extract keywords keywords = query.lower().split() # Find matching signal types matching_signals = [] all_signal_types = set(self.graph.signal_type_index.keys()) for keyword in keywords: for signal_type in all_signal_types: if keyword in signal_type: matching_signals.append(signal_type) if not matching_signals: return { "status": "unknown_query", "query": query, "message": "Could not infer index from query. Try specifying signal types or use a known index name.", "available_signals": list(all_signal_types)[:20] } # Build ad-hoc index return self._build_ad_hoc_index(query, matching_signals) def _build_ad_hoc_index(self, name: str, signal_types: List[str]) -> Dict: """Build an ad-hoc index from available signals""" available_signals = [] for signal_type in signal_types: signals = self.graph.find_by_type(signal_type) if signals: avg_value = sum(s["value"] for s in signals) / len(signals) avg_confidence = sum(s.get("confidence", 0.5) for s in signals) / len(signals) available_signals.append({ "signal_type": signal_type, "value": avg_value, "weight": 1.0 / len(signal_types), # Equal weights "confidence": avg_confidence, "sample_size": len(signals) }) if not available_signals: return { "status": "insufficient_data", "message": "No matching signals found" } score = sum(s["value"] * s["weight"] for s in available_signals) confidence = sum(s["confidence"] * s["weight"] for s in available_signals) return { "status": "success", "index_name": f"ad_hoc_{name[:30]}", "score": round(score, 2), "confidence": round(confidence, 2), "signals_used": len(available_signals), "component_signals": available_signals, "interpretation": f"Ad-hoc index based on {len(available_signals)} signal types" } def discover_new_index(self, target_variable: str) -> Dict: """ Discover what signals correlate with a target variable Example: "What signals correlate with commercial success?" """ # Find correlations correlations = self.graph.discover_correlations() # Filter for target-related correlations target_correlations = [ c for c in correlations if target_variable.lower() in c["type_a"].lower() or target_variable.lower() in c["type_b"].lower() ] if not target_correlations: return { "status": "no_correlations", "target": target_variable, "message": f"No strong correlations found with {target_variable}" } # Build index from top correlations top_signals = [] for corr in target_correlations[:5]: signal_type = corr["type_b"] if target_variable.lower() in corr["type_a"].lower() else corr["type_a"] top_signals.append({ "signal_type": signal_type, "correlation": corr["correlation"], "sample_size": corr["sample_size"] }) return { "status": "success", "target": target_variable, "discovered_signals": top_signals, "message": f"Found {len(top_signals)} signals that correlate with {target_variable}" } def _interpret_score(self, index_name: str, score: float) -> str: """Interpret an index score""" interpretations = { "family_friendly": { 20: "Not suitable for families", 40: "Limited family amenities", 60: "Moderately family-friendly", 80: "Family-friendly", 100: "Excellent for families" }, "restaurant_success": { 20: "Poor location for restaurants", 40: "Challenging conditions", 60: "Moderate potential", 80: "Good location", 100: "Excellent for restaurants" }, "investment_readiness": { 20: "High risk", 40: "Significant issues", 60: "Moderate readiness", 80: "Good readiness", 100: "Excellent investment opportunity" } } interp = interpretations.get(index_name, {}) for threshold, text in sorted(interp.items()): if score <= threshold: return text return interp.get(100, f"Score: {score:.1f}") # Example usage def example_semantic_graph(): """Example: Build semantic graph and query it""" graph = SemanticGraph() # Add signals signals = [ {"signal_type": "traffic_intensity", "category": "urban", "value": 75, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}}, {"signal_type": "sidewalk_width", "category": "urban", "value": 60, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}}, {"signal_type": "noise_level", "category": "urban", "value": 80, "unit": "db", "location": {"lat": 13.7563, "lng": 100.5018}}, {"signal_type": "playground", "category": "social", "value": 20, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}}, {"signal_type": "greenery", "category": "urban", "value": 45, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}}, {"signal_type": "lighting", "category": "infrastructure", "value": 70, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}}, {"signal_type": "pedestrian_flow", "category": "social", "value": 85, "unit": "people_per_hour", "location": {"lat": 13.7563, "lng": 100.5018}}, {"signal_type": "evening_lighting", "category": "infrastructure", "value": 65, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}}, {"signal_type": "dwell_time", "category": "social", "value": 40, "unit": "minutes", "location": {"lat": 13.7563, "lng": 100.5018}}, ] for signal in signals: graph.add_signal(signal) # Add relationships graph.add_relationship( graph.signal_type_index["traffic_intensity"][0], graph.signal_type_index["noise_level"][0], "causes", 0.85 ) print("=== Semantic Graph ===") stats = graph.get_stats() print(f"Nodes: {stats['total_nodes']}, Edges: {stats['total_edges']}") print(f"Signal types: {stats['signal_types']}") # AI Reasoning ai = AIReasoningLayer(graph) # Query 1: Family-friendly print("\n=== Query: Family-friendly ===") result = ai.answer_query("How attractive is this area for families with children?") print(f"Score: {result.get('score', 'N/A')}") print(f"Confidence: {result.get('confidence', 'N/A')}") print(f"Signals used: {result.get('signals_used', 0)}/{result.get('signals_needed', 0)}") print(f"Interpretation: {result.get('interpretation', 'N/A')}") # Query 2: Restaurant success print("\n=== Query: Restaurant Success ===") result = ai.answer_query("What is the commercial potential for restaurants here?") print(f"Score: {result.get('score', 'N/A')}") print(f"Interpretation: {result.get('interpretation', 'N/A')}") # Discover correlations print("\n=== Discover Correlations ===") correlations = graph.discover_correlations() for corr in correlations[:3]: print(f" {corr['type_a']} ↔ {corr['type_b']}: {corr['correlation']:.2f}") return graph, ai if __name__ == '__main__': example_semantic_graph()