Bernt
bae705aa97
- Add NFC ePassport roadmap (ICAO 9303, eIDAS) - Add TensorFlow.js edge face detection (BlazeFace) - Add structured audit logger (GDPR-compliant) - Risk scoring support Part of KYC Apple Native UX v1.1.0
519 lines
21 KiB
Python
519 lines
21 KiB
Python
"""
|
|
Semantic Graph — Connects Reality Signals into a knowledge graph
|
|
Enables AI to discover relationships and build custom indexes
|
|
"""
|
|
|
|
from typing import Dict, List, Optional, Set, Tuple
|
|
from dataclasses import dataclass
|
|
from datetime import datetime
|
|
import json
|
|
|
|
|
|
@dataclass
|
|
class SignalNode:
|
|
"""A node in the semantic graph"""
|
|
signal_id: str
|
|
signal_type: str
|
|
category: str
|
|
value: float
|
|
unit: str
|
|
location: Optional[Dict]
|
|
timestamp: str
|
|
relationships: List[str] # IDs of related signals
|
|
|
|
def to_dict(self) -> Dict:
|
|
return {
|
|
"signal_id": self.signal_id,
|
|
"signal_type": self.signal_type,
|
|
"category": self.category,
|
|
"value": round(self.value, 2),
|
|
"unit": self.unit,
|
|
"location": self.location,
|
|
"timestamp": self.timestamp,
|
|
"relationships": self.relationships
|
|
}
|
|
|
|
|
|
class SemanticGraph:
|
|
"""Knowledge graph for reality signals"""
|
|
|
|
def __init__(self):
|
|
self.nodes: Dict[str, SignalNode] = {}
|
|
self.edges: List[Dict] = []
|
|
self.signal_type_index: Dict[str, List[str]] = {} # type -> signal_ids
|
|
self.location_index: Dict[str, List[str]] = {} # location_hash -> signal_ids
|
|
|
|
def add_signal(self, signal: Dict) -> str:
|
|
"""Add a signal to the graph"""
|
|
signal_id = f"{signal['signal_type']}_{signal.get('timestamp', datetime.utcnow().isoformat())}"
|
|
|
|
node = SignalNode(
|
|
signal_id=signal_id,
|
|
signal_type=signal['signal_type'],
|
|
category=signal['category'],
|
|
value=signal['value'],
|
|
unit=signal['unit'],
|
|
location=signal.get('location'),
|
|
timestamp=signal.get('timestamp', datetime.utcnow().isoformat()),
|
|
relationships=[]
|
|
)
|
|
|
|
self.nodes[signal_id] = node
|
|
|
|
# Index by type
|
|
if node.signal_type not in self.signal_type_index:
|
|
self.signal_type_index[node.signal_type] = []
|
|
self.signal_type_index[node.signal_type].append(signal_id)
|
|
|
|
# Index by location
|
|
if node.location:
|
|
loc_hash = self._hash_location(node.location)
|
|
if loc_hash not in self.location_index:
|
|
self.location_index[loc_hash] = []
|
|
self.location_index[loc_hash].append(signal_id)
|
|
|
|
return signal_id
|
|
|
|
def add_relationship(self, source_id: str, target_id: str, relation_type: str, strength: float):
|
|
"""Add a relationship between two signals"""
|
|
if source_id in self.nodes and target_id in self.nodes:
|
|
self.edges.append({
|
|
"source": source_id,
|
|
"target": target_id,
|
|
"type": relation_type,
|
|
"strength": round(strength, 2)
|
|
})
|
|
|
|
# Update node relationships
|
|
self.nodes[source_id].relationships.append(target_id)
|
|
self.nodes[target_id].relationships.append(source_id)
|
|
|
|
def find_related(self, signal_id: str, relation_type: Optional[str] = None) -> List[Dict]:
|
|
"""Find signals related to a given signal"""
|
|
if signal_id not in self.nodes:
|
|
return []
|
|
|
|
related = []
|
|
for edge in self.edges:
|
|
if edge["source"] == signal_id or edge["target"] == signal_id:
|
|
if relation_type is None or edge["type"] == relation_type:
|
|
other_id = edge["target"] if edge["source"] == signal_id else edge["source"]
|
|
if other_id in self.nodes:
|
|
related.append({
|
|
"signal": self.nodes[other_id].to_dict(),
|
|
"relationship": edge["type"],
|
|
"strength": edge["strength"]
|
|
})
|
|
|
|
return related
|
|
|
|
def find_by_type(self, signal_type: str) -> List[Dict]:
|
|
"""Find all signals of a given type"""
|
|
signal_ids = self.signal_type_index.get(signal_type, [])
|
|
return [self.nodes[sid].to_dict() for sid in signal_ids if sid in self.nodes]
|
|
|
|
def find_by_location(self, location: Dict, radius_meters: float = 100) -> List[Dict]:
|
|
"""Find signals near a location"""
|
|
# Simplified: exact location hash match
|
|
# In production: use PostGIS spatial query
|
|
loc_hash = self._hash_location(location)
|
|
signal_ids = self.location_index.get(loc_hash, [])
|
|
return [self.nodes[sid].to_dict() for sid in signal_ids if sid in self.nodes]
|
|
|
|
def get_subgraph(self, signal_types: List[str]) -> Dict:
|
|
"""Get a subgraph containing only specified signal types"""
|
|
nodes = {}
|
|
edges = []
|
|
|
|
for signal_type in signal_types:
|
|
for signal_id in self.signal_type_index.get(signal_type, []):
|
|
if signal_id in self.nodes:
|
|
nodes[signal_id] = self.nodes[signal_id].to_dict()
|
|
|
|
# Find edges between these nodes
|
|
node_ids = set(nodes.keys())
|
|
for edge in self.edges:
|
|
if edge["source"] in node_ids and edge["target"] in node_ids:
|
|
edges.append(edge)
|
|
|
|
return {
|
|
"nodes": list(nodes.values()),
|
|
"edges": edges,
|
|
"node_count": len(nodes),
|
|
"edge_count": len(edges)
|
|
}
|
|
|
|
def discover_correlations(self) -> List[Dict]:
|
|
"""Discover correlations between signal types"""
|
|
correlations = []
|
|
|
|
# Get all signal types
|
|
types = list(self.signal_type_index.keys())
|
|
|
|
# Check correlations between pairs
|
|
for i, type_a in enumerate(types):
|
|
for type_b in types[i+1:]:
|
|
signals_a = [self.nodes[sid] for sid in self.signal_type_index.get(type_a, []) if sid in self.nodes]
|
|
signals_b = [self.nodes[sid] for sid in self.signal_type_index.get(type_b, []) if sid in self.nodes]
|
|
|
|
if signals_a and signals_b:
|
|
# Simple correlation: compare averages
|
|
avg_a = sum(s.value for s in signals_a) / len(signals_a)
|
|
avg_b = sum(s.value for s in signals_b) / len(signals_b)
|
|
|
|
# Correlation strength (simplified)
|
|
correlation = 1 - abs(avg_a - avg_b) / max(avg_a, avg_b, 1)
|
|
|
|
if correlation > 0.5: # Threshold
|
|
correlations.append({
|
|
"type_a": type_a,
|
|
"type_b": type_b,
|
|
"correlation": round(correlation, 2),
|
|
"avg_a": round(avg_a, 2),
|
|
"avg_b": round(avg_b, 2),
|
|
"sample_size": min(len(signals_a), len(signals_b))
|
|
})
|
|
|
|
# Sort by correlation strength
|
|
correlations.sort(key=lambda x: x["correlation"], reverse=True)
|
|
|
|
return correlations
|
|
|
|
def _hash_location(self, location: Dict) -> str:
|
|
"""Hash location for indexing"""
|
|
lat = location.get("lat", location.get("latitude", 0))
|
|
lng = location.get("lng", location.get("longitude", 0))
|
|
# Round to ~100m precision
|
|
return f"{round(lat, 3)}_{round(lng, 3)}"
|
|
|
|
def get_stats(self) -> Dict:
|
|
"""Get graph statistics"""
|
|
return {
|
|
"total_nodes": len(self.nodes),
|
|
"total_edges": len(self.edges),
|
|
"signal_types": len(self.signal_type_index),
|
|
"locations": len(self.location_index),
|
|
"type_distribution": {
|
|
stype: len(sids)
|
|
for stype, sids in self.signal_type_index.items()
|
|
}
|
|
}
|
|
|
|
|
|
class AIReasoningLayer:
|
|
"""AI reasoning layer for dynamic index creation"""
|
|
|
|
def __init__(self, semantic_graph: SemanticGraph):
|
|
self.graph = semantic_graph
|
|
self.index_templates = self._load_index_templates()
|
|
|
|
def _load_index_templates(self) -> Dict:
|
|
"""Load index templates for common queries"""
|
|
return {
|
|
"family_friendly": {
|
|
"signals": ["traffic_intensity", "sidewalk_width", "noise_level", "air_quality", "playground", "greenery", "schools", "lighting", "crosswalks", "accessibility"],
|
|
"weights": [0.15, 0.10, 0.10, 0.10, 0.15, 0.10, 0.10, 0.10, 0.05, 0.05],
|
|
"description": "Measures attractiveness for families with children"
|
|
},
|
|
"restaurant_success": {
|
|
"signals": ["pedestrian_flow", "evening_lighting", "shade_coverage", "public_transit", "dwell_time", "competition_density", "rent_burden", "parking"],
|
|
"weights": [0.20, 0.15, 0.10, 0.15, 0.15, 0.10, 0.10, 0.05],
|
|
"description": "Predicts commercial success for restaurants"
|
|
},
|
|
"investment_readiness": {
|
|
"signals": ["infrastructure_reliability", "maintenance_quality", "urban_contradiction", "informal_economy", "mobile_connectivity", "road_integrity", "street_lighting_reliability"],
|
|
"weights": [0.20, 0.15, 0.15, 0.10, 0.15, 0.15, 0.10],
|
|
"description": "Measures readiness for infrastructure investment"
|
|
},
|
|
"tourism_potential": {
|
|
"signals": ["public_life", "night_activity", "walkability", "shade_index", "public_toilet_index", "wifi_availability", "safety_index", "visual_maintenance"],
|
|
"weights": [0.15, 0.15, 0.15, 0.10, 0.10, 0.10, 0.15, 0.10],
|
|
"description": "Measures tourism attractiveness"
|
|
},
|
|
"climate_resilience": {
|
|
"signals": ["heat_stress", "shade_index", "drainage_performance", "tree_coverage", "surface_temperature", "green_space", "water_retention"],
|
|
"weights": [0.20, 0.15, 0.20, 0.15, 0.15, 0.10, 0.05],
|
|
"description": "Measures urban climate resilience"
|
|
}
|
|
}
|
|
|
|
def answer_query(self, query: str) -> Dict:
|
|
"""
|
|
Answer a natural language query by building a custom index
|
|
|
|
Examples:
|
|
- "How attractive is this area for families?"
|
|
- "What signals correlate with restaurant success?"
|
|
- "Show me a climate resilience index"
|
|
"""
|
|
# Parse query
|
|
query_lower = query.lower()
|
|
|
|
# Match to template
|
|
matched_template = None
|
|
for template_name, template in self.index_templates.items():
|
|
if any(keyword in query_lower for keyword in template_name.split("_")):
|
|
matched_template = template_name
|
|
break
|
|
|
|
if not matched_template:
|
|
# Try to infer from query
|
|
return self._infer_index_from_query(query)
|
|
|
|
# Build index from template
|
|
template = self.index_templates[matched_template]
|
|
return self._build_index_from_template(matched_template, template)
|
|
|
|
def _build_index_from_template(self, name: str, template: Dict) -> Dict:
|
|
"""Build an index from a template"""
|
|
signals_needed = template["signals"]
|
|
weights = template["weights"]
|
|
|
|
# Find available signals
|
|
available_signals = []
|
|
missing_signals = []
|
|
|
|
for signal_type, weight in zip(signals_needed, weights):
|
|
signals = self.graph.find_by_type(signal_type)
|
|
if signals:
|
|
# Average value
|
|
avg_value = sum(s["value"] for s in signals) / len(signals)
|
|
avg_confidence = sum(s.get("confidence", 0.5) for s in signals) / len(signals)
|
|
|
|
available_signals.append({
|
|
"signal_type": signal_type,
|
|
"value": avg_value,
|
|
"weight": weight,
|
|
"confidence": avg_confidence,
|
|
"sample_size": len(signals)
|
|
})
|
|
else:
|
|
missing_signals.append(signal_type)
|
|
|
|
if not available_signals:
|
|
return {
|
|
"status": "insufficient_data",
|
|
"index_name": name,
|
|
"missing_signals": missing_signals,
|
|
"message": "No required signals available in the graph"
|
|
}
|
|
|
|
# Calculate weighted score
|
|
total_weight = sum(s["weight"] for s in available_signals)
|
|
if total_weight == 0:
|
|
total_weight = 1
|
|
|
|
score = sum(s["value"] * s["weight"] for s in available_signals) / total_weight
|
|
confidence = sum(s["confidence"] * s["weight"] for s in available_signals) / total_weight
|
|
|
|
return {
|
|
"status": "success",
|
|
"index_name": name,
|
|
"score": round(score, 2),
|
|
"confidence": round(confidence, 2),
|
|
"signals_used": len(available_signals),
|
|
"signals_needed": len(signals_needed),
|
|
"missing_signals": missing_signals,
|
|
"component_signals": available_signals,
|
|
"description": template["description"],
|
|
"interpretation": self._interpret_score(name, score)
|
|
}
|
|
|
|
def _infer_index_from_query(self, query: str) -> Dict:
|
|
"""Infer index requirements from query"""
|
|
# Extract keywords
|
|
keywords = query.lower().split()
|
|
|
|
# Find matching signal types
|
|
matching_signals = []
|
|
all_signal_types = set(self.graph.signal_type_index.keys())
|
|
|
|
for keyword in keywords:
|
|
for signal_type in all_signal_types:
|
|
if keyword in signal_type:
|
|
matching_signals.append(signal_type)
|
|
|
|
if not matching_signals:
|
|
return {
|
|
"status": "unknown_query",
|
|
"query": query,
|
|
"message": "Could not infer index from query. Try specifying signal types or use a known index name.",
|
|
"available_signals": list(all_signal_types)[:20]
|
|
}
|
|
|
|
# Build ad-hoc index
|
|
return self._build_ad_hoc_index(query, matching_signals)
|
|
|
|
def _build_ad_hoc_index(self, name: str, signal_types: List[str]) -> Dict:
|
|
"""Build an ad-hoc index from available signals"""
|
|
available_signals = []
|
|
|
|
for signal_type in signal_types:
|
|
signals = self.graph.find_by_type(signal_type)
|
|
if signals:
|
|
avg_value = sum(s["value"] for s in signals) / len(signals)
|
|
avg_confidence = sum(s.get("confidence", 0.5) for s in signals) / len(signals)
|
|
|
|
available_signals.append({
|
|
"signal_type": signal_type,
|
|
"value": avg_value,
|
|
"weight": 1.0 / len(signal_types), # Equal weights
|
|
"confidence": avg_confidence,
|
|
"sample_size": len(signals)
|
|
})
|
|
|
|
if not available_signals:
|
|
return {
|
|
"status": "insufficient_data",
|
|
"message": "No matching signals found"
|
|
}
|
|
|
|
score = sum(s["value"] * s["weight"] for s in available_signals)
|
|
confidence = sum(s["confidence"] * s["weight"] for s in available_signals)
|
|
|
|
return {
|
|
"status": "success",
|
|
"index_name": f"ad_hoc_{name[:30]}",
|
|
"score": round(score, 2),
|
|
"confidence": round(confidence, 2),
|
|
"signals_used": len(available_signals),
|
|
"component_signals": available_signals,
|
|
"interpretation": f"Ad-hoc index based on {len(available_signals)} signal types"
|
|
}
|
|
|
|
def discover_new_index(self, target_variable: str) -> Dict:
|
|
"""
|
|
Discover what signals correlate with a target variable
|
|
|
|
Example: "What signals correlate with commercial success?"
|
|
"""
|
|
# Find correlations
|
|
correlations = self.graph.discover_correlations()
|
|
|
|
# Filter for target-related correlations
|
|
target_correlations = [
|
|
c for c in correlations
|
|
if target_variable.lower() in c["type_a"].lower() or target_variable.lower() in c["type_b"].lower()
|
|
]
|
|
|
|
if not target_correlations:
|
|
return {
|
|
"status": "no_correlations",
|
|
"target": target_variable,
|
|
"message": f"No strong correlations found with {target_variable}"
|
|
}
|
|
|
|
# Build index from top correlations
|
|
top_signals = []
|
|
for corr in target_correlations[:5]:
|
|
signal_type = corr["type_b"] if target_variable.lower() in corr["type_a"].lower() else corr["type_a"]
|
|
top_signals.append({
|
|
"signal_type": signal_type,
|
|
"correlation": corr["correlation"],
|
|
"sample_size": corr["sample_size"]
|
|
})
|
|
|
|
return {
|
|
"status": "success",
|
|
"target": target_variable,
|
|
"discovered_signals": top_signals,
|
|
"message": f"Found {len(top_signals)} signals that correlate with {target_variable}"
|
|
}
|
|
|
|
def _interpret_score(self, index_name: str, score: float) -> str:
|
|
"""Interpret an index score"""
|
|
interpretations = {
|
|
"family_friendly": {
|
|
20: "Not suitable for families",
|
|
40: "Limited family amenities",
|
|
60: "Moderately family-friendly",
|
|
80: "Family-friendly",
|
|
100: "Excellent for families"
|
|
},
|
|
"restaurant_success": {
|
|
20: "Poor location for restaurants",
|
|
40: "Challenging conditions",
|
|
60: "Moderate potential",
|
|
80: "Good location",
|
|
100: "Excellent for restaurants"
|
|
},
|
|
"investment_readiness": {
|
|
20: "High risk",
|
|
40: "Significant issues",
|
|
60: "Moderate readiness",
|
|
80: "Good readiness",
|
|
100: "Excellent investment opportunity"
|
|
}
|
|
}
|
|
|
|
interp = interpretations.get(index_name, {})
|
|
for threshold, text in sorted(interp.items()):
|
|
if score <= threshold:
|
|
return text
|
|
|
|
return interp.get(100, f"Score: {score:.1f}")
|
|
|
|
|
|
# Example usage
|
|
def example_semantic_graph():
|
|
"""Example: Build semantic graph and query it"""
|
|
graph = SemanticGraph()
|
|
|
|
# Add signals
|
|
signals = [
|
|
{"signal_type": "traffic_intensity", "category": "urban", "value": 75, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}},
|
|
{"signal_type": "sidewalk_width", "category": "urban", "value": 60, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}},
|
|
{"signal_type": "noise_level", "category": "urban", "value": 80, "unit": "db", "location": {"lat": 13.7563, "lng": 100.5018}},
|
|
{"signal_type": "playground", "category": "social", "value": 20, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}},
|
|
{"signal_type": "greenery", "category": "urban", "value": 45, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}},
|
|
{"signal_type": "lighting", "category": "infrastructure", "value": 70, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}},
|
|
{"signal_type": "pedestrian_flow", "category": "social", "value": 85, "unit": "people_per_hour", "location": {"lat": 13.7563, "lng": 100.5018}},
|
|
{"signal_type": "evening_lighting", "category": "infrastructure", "value": 65, "unit": "score", "location": {"lat": 13.7563, "lng": 100.5018}},
|
|
{"signal_type": "dwell_time", "category": "social", "value": 40, "unit": "minutes", "location": {"lat": 13.7563, "lng": 100.5018}},
|
|
]
|
|
|
|
for signal in signals:
|
|
graph.add_signal(signal)
|
|
|
|
# Add relationships
|
|
graph.add_relationship(
|
|
graph.signal_type_index["traffic_intensity"][0],
|
|
graph.signal_type_index["noise_level"][0],
|
|
"causes",
|
|
0.85
|
|
)
|
|
|
|
print("=== Semantic Graph ===")
|
|
stats = graph.get_stats()
|
|
print(f"Nodes: {stats['total_nodes']}, Edges: {stats['total_edges']}")
|
|
print(f"Signal types: {stats['signal_types']}")
|
|
|
|
# AI Reasoning
|
|
ai = AIReasoningLayer(graph)
|
|
|
|
# Query 1: Family-friendly
|
|
print("\n=== Query: Family-friendly ===")
|
|
result = ai.answer_query("How attractive is this area for families with children?")
|
|
print(f"Score: {result.get('score', 'N/A')}")
|
|
print(f"Confidence: {result.get('confidence', 'N/A')}")
|
|
print(f"Signals used: {result.get('signals_used', 0)}/{result.get('signals_needed', 0)}")
|
|
print(f"Interpretation: {result.get('interpretation', 'N/A')}")
|
|
|
|
# Query 2: Restaurant success
|
|
print("\n=== Query: Restaurant Success ===")
|
|
result = ai.answer_query("What is the commercial potential for restaurants here?")
|
|
print(f"Score: {result.get('score', 'N/A')}")
|
|
print(f"Interpretation: {result.get('interpretation', 'N/A')}")
|
|
|
|
# Discover correlations
|
|
print("\n=== Discover Correlations ===")
|
|
correlations = graph.discover_correlations()
|
|
for corr in correlations[:3]:
|
|
print(f" {corr['type_a']} ↔ {corr['type_b']}: {corr['correlation']:.2f}")
|
|
|
|
return graph, ai
|
|
|
|
|
|
if __name__ == '__main__':
|
|
example_semantic_graph()
|