Bernt
bae705aa97
- Add NFC ePassport roadmap (ICAO 9303, eIDAS) - Add TensorFlow.js edge face detection (BlazeFace) - Add structured audit logger (GDPR-compliant) - Risk scoring support Part of KYC Apple Native UX v1.1.0
571 lines
20 KiB
Python
571 lines
20 KiB
Python
"""
|
|
Observation Store - Database layer for IOM observations
|
|
Supports PostgreSQL with PostGIS for geospatial queries
|
|
"""
|
|
|
|
import json
|
|
import os
|
|
from datetime import datetime, timedelta
|
|
from typing import List, Optional, Dict, Any
|
|
from contextlib import contextmanager
|
|
|
|
# Database imports
|
|
import psycopg2
|
|
from psycopg2.extras import RealDictCursor
|
|
from psycopg2.pool import ThreadedConnectionPool
|
|
|
|
from observation_models import Observation, ObservationSummary, ObservationTrend
|
|
|
|
|
|
class ObservationStore:
|
|
"""Store and retrieve observations from database"""
|
|
|
|
def __init__(self, dsn: Optional[str] = None):
|
|
"""
|
|
Initialize observation store
|
|
|
|
Args:
|
|
dsn: PostgreSQL connection string. If None, uses environment variable.
|
|
"""
|
|
if dsn is None:
|
|
dsn = os.environ.get(
|
|
'DATABASE_URL',
|
|
'postgresql://localhost:5432/iom'
|
|
)
|
|
|
|
self.dsn = dsn
|
|
self.pool = None
|
|
self._init_pool()
|
|
self._init_tables()
|
|
|
|
def _init_pool(self):
|
|
"""Initialize connection pool"""
|
|
self.pool = ThreadedConnectionPool(
|
|
minconn=1,
|
|
maxconn=10,
|
|
dsn=self.dsn
|
|
)
|
|
|
|
@contextmanager
|
|
def _get_connection(self):
|
|
"""Get connection from pool"""
|
|
conn = self.pool.getconn()
|
|
try:
|
|
yield conn
|
|
finally:
|
|
self.pool.putconn(conn)
|
|
|
|
@contextmanager
|
|
def _get_cursor(self, conn):
|
|
"""Get cursor from connection"""
|
|
cursor = conn.cursor(cursor_factory=RealDictCursor)
|
|
try:
|
|
yield cursor
|
|
finally:
|
|
cursor.close()
|
|
|
|
def _init_tables(self):
|
|
"""Initialize database tables"""
|
|
with self._get_connection() as conn:
|
|
with self._get_cursor(conn) as cur:
|
|
# Enable PostGIS
|
|
cur.execute("CREATE EXTENSION IF NOT EXISTS postgis;")
|
|
|
|
# Observations table
|
|
cur.execute("""
|
|
CREATE TABLE IF NOT EXISTS observations (
|
|
id VARCHAR(20) PRIMARY KEY,
|
|
timestamp TIMESTAMPTZ NOT NULL,
|
|
object_goid VARCHAR(50) NOT NULL,
|
|
observer VARCHAR(50) NOT NULL,
|
|
findings JSONB NOT NULL DEFAULT '[]',
|
|
media JSONB NOT NULL DEFAULT '[]',
|
|
weather JSONB,
|
|
ai_analysis JSONB,
|
|
location GEOGRAPHY(POINT, 4326),
|
|
created_at TIMESTAMPTZ DEFAULT NOW(),
|
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
""")
|
|
|
|
# Indexes
|
|
cur.execute("""
|
|
CREATE INDEX IF NOT EXISTS idx_observations_object
|
|
ON observations(object_goid);
|
|
""")
|
|
|
|
cur.execute("""
|
|
CREATE INDEX IF NOT EXISTS idx_observations_timestamp
|
|
ON observations(timestamp DESC);
|
|
""")
|
|
|
|
cur.execute("""
|
|
CREATE INDEX IF NOT EXISTS idx_observations_location
|
|
ON observations USING GIST(location);
|
|
""")
|
|
|
|
cur.execute("""
|
|
CREATE INDEX IF NOT EXISTS idx_observations_findings
|
|
ON observations USING GIN(findings);
|
|
""")
|
|
|
|
# Object state table (latest condition per object)
|
|
cur.execute("""
|
|
CREATE TABLE IF NOT EXISTS object_states (
|
|
object_goid VARCHAR(50) PRIMARY KEY,
|
|
latest_condition INTEGER,
|
|
latest_observation_id VARCHAR(20),
|
|
latest_observation_date TIMESTAMPTZ,
|
|
observation_count INTEGER DEFAULT 0,
|
|
finding_types JSONB DEFAULT '[]',
|
|
risk_level DECIMAL(3,1),
|
|
next_observation_due TIMESTAMPTZ,
|
|
updated_at TIMESTAMPTZ DEFAULT NOW()
|
|
);
|
|
""")
|
|
|
|
conn.commit()
|
|
|
|
def _generate_id(self) -> str:
|
|
"""Generate unique observation ID"""
|
|
year = datetime.now().year
|
|
|
|
with self._get_connection() as conn:
|
|
with self._get_cursor(conn) as cur:
|
|
cur.execute("""
|
|
SELECT COUNT(*) as count
|
|
FROM observations
|
|
WHERE id LIKE %s
|
|
""", (f'OBS-{year}-%',))
|
|
|
|
result = cur.fetchone()
|
|
seq = (result['count'] + 1) if result else 1
|
|
|
|
return f"OBS-{year}-{seq:07d}"
|
|
|
|
def create(self, observation: Observation) -> str:
|
|
"""
|
|
Save observation to database
|
|
|
|
Args:
|
|
observation: Observation object to save
|
|
|
|
Returns:
|
|
Observation ID
|
|
"""
|
|
# Generate ID if not provided
|
|
if not observation.id:
|
|
observation.id = self._generate_id()
|
|
|
|
# Extract location from first media geotag
|
|
location = None
|
|
if observation.media and observation.media[0].geotag:
|
|
geo = observation.media[0].geotag
|
|
location = f"POINT({geo.lng} {geo.lat})"
|
|
|
|
with self._get_connection() as conn:
|
|
with self._get_cursor(conn) as cur:
|
|
# Insert observation
|
|
cur.execute("""
|
|
INSERT INTO observations (
|
|
id, timestamp, object_goid, observer,
|
|
findings, media, weather, ai_analysis, location
|
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s::geography)
|
|
""", (
|
|
observation.id,
|
|
observation.timestamp,
|
|
observation.object_goid,
|
|
observation.observer,
|
|
json.dumps([f.model_dump() for f in observation.findings]),
|
|
json.dumps([m.model_dump() for m in observation.media]),
|
|
json.dumps(observation.weather.model_dump()) if observation.weather else None,
|
|
json.dumps(observation.ai_analysis.model_dump()) if observation.ai_analysis else None,
|
|
location
|
|
))
|
|
|
|
# Update object state
|
|
self._update_object_state(conn, observation)
|
|
|
|
conn.commit()
|
|
|
|
return observation.id
|
|
|
|
def _update_object_state(self, conn, observation: Observation):
|
|
"""Update object state with latest observation"""
|
|
with self._get_cursor(conn) as cur:
|
|
# Extract finding types
|
|
finding_types = list(set(f.type.value for f in observation.findings))
|
|
|
|
# Extract risk level from AI analysis
|
|
risk_level = None
|
|
if observation.ai_analysis:
|
|
# Map condition to risk (simplified)
|
|
condition = observation.ai_analysis.overall_condition
|
|
risk_level = condition * 2 # 1-5 -> 2-10
|
|
|
|
# Upsert object state
|
|
cur.execute("""
|
|
INSERT INTO object_states (
|
|
object_goid, latest_condition, latest_observation_id,
|
|
latest_observation_date, observation_count, finding_types,
|
|
risk_level, next_observation_due
|
|
) VALUES (%s, %s, %s, %s, 1, %s, %s, %s)
|
|
ON CONFLICT (object_goid) DO UPDATE SET
|
|
latest_condition = EXCLUDED.latest_condition,
|
|
latest_observation_id = EXCLUDED.latest_observation_id,
|
|
latest_observation_date = EXCLUDED.latest_observation_date,
|
|
observation_count = object_states.observation_count + 1,
|
|
finding_types = EXCLUDED.finding_types,
|
|
risk_level = EXCLUDED.risk_level,
|
|
next_observation_due = EXCLUDED.next_observation_due,
|
|
updated_at = NOW();
|
|
""", (
|
|
observation.object_goid,
|
|
observation.ai_analysis.overall_condition if observation.ai_analysis else None,
|
|
observation.id,
|
|
observation.timestamp,
|
|
json.dumps(finding_types),
|
|
risk_level,
|
|
observation.ai_analysis.next_observation_due if observation.ai_analysis else None
|
|
))
|
|
|
|
def get(self, observation_id: str) -> Optional[Observation]:
|
|
"""
|
|
Get observation by ID
|
|
|
|
Args:
|
|
observation_id: Observation ID
|
|
|
|
Returns:
|
|
Observation object or None
|
|
"""
|
|
with self._get_connection() as conn:
|
|
with self._get_cursor(conn) as cur:
|
|
cur.execute("""
|
|
SELECT * FROM observations WHERE id = %s
|
|
""", (observation_id,))
|
|
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
|
|
return self._row_to_observation(row)
|
|
|
|
def get_for_object(
|
|
self,
|
|
goid: str,
|
|
limit: int = 100,
|
|
offset: int = 0
|
|
) -> List[Observation]:
|
|
"""
|
|
Get all observations for an object
|
|
|
|
Args:
|
|
goid: Object GOID
|
|
limit: Maximum number of observations
|
|
offset: Offset for pagination
|
|
|
|
Returns:
|
|
List of observations
|
|
"""
|
|
with self._get_connection() as conn:
|
|
with self._get_cursor(conn) as cur:
|
|
cur.execute("""
|
|
SELECT * FROM observations
|
|
WHERE object_goid = %s
|
|
ORDER BY timestamp DESC
|
|
LIMIT %s OFFSET %s
|
|
""", (goid, limit, offset))
|
|
|
|
rows = cur.fetchall()
|
|
return [self._row_to_observation(row) for row in rows]
|
|
|
|
def get_summary(self, goid: str) -> Optional[ObservationSummary]:
|
|
"""
|
|
Get observation summary for an object
|
|
|
|
Args:
|
|
goid: Object GOID
|
|
|
|
Returns:
|
|
Observation summary or None
|
|
"""
|
|
with self._get_connection() as conn:
|
|
with self._get_cursor(conn) as cur:
|
|
cur.execute("""
|
|
SELECT * FROM object_states WHERE object_goid = %s
|
|
""", (goid,))
|
|
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
|
|
return ObservationSummary(
|
|
object_goid=row['object_goid'],
|
|
observation_count=row['observation_count'],
|
|
latest_condition=row['latest_condition'],
|
|
latest_observation_date=row['latest_observation_date'],
|
|
finding_types=json.loads(row['finding_types']) if row['finding_types'] else [],
|
|
risk_trend=self._calculate_trend(goid)
|
|
)
|
|
|
|
def get_trend(self, goid: str, months: int = 6) -> ObservationTrend:
|
|
"""
|
|
Get trend analysis for an object
|
|
|
|
Args:
|
|
goid: Object GOID
|
|
months: Number of months to analyze
|
|
|
|
Returns:
|
|
Observation trend
|
|
"""
|
|
with self._get_connection() as conn:
|
|
with self._get_cursor(conn) as cur:
|
|
cur.execute("""
|
|
SELECT
|
|
DATE_TRUNC('month', timestamp) as month,
|
|
AVG((ai_analysis->>'overall_condition')::int) as avg_condition,
|
|
COUNT(*) as observation_count
|
|
FROM observations
|
|
WHERE object_goid = %s
|
|
AND timestamp >= NOW() - INTERVAL '%s months'
|
|
GROUP BY DATE_TRUNC('month', timestamp)
|
|
ORDER BY month
|
|
""", (goid, months))
|
|
|
|
rows = cur.fetchall()
|
|
|
|
data_points = [
|
|
{
|
|
'month': row['month'].strftime('%Y-%m'),
|
|
'condition': round(row['avg_condition'], 1) if row['avg_condition'] else None,
|
|
'observations': row['observation_count']
|
|
}
|
|
for row in rows
|
|
]
|
|
|
|
return ObservationTrend(
|
|
object_goid=goid,
|
|
period_months=months,
|
|
data_points=data_points
|
|
)
|
|
|
|
def find_by_location(
|
|
self,
|
|
lat: float,
|
|
lng: float,
|
|
radius_meters: float,
|
|
limit: int = 100
|
|
) -> List[Observation]:
|
|
"""
|
|
Find observations near a location
|
|
|
|
Args:
|
|
lat: Latitude
|
|
lng: Longitude
|
|
radius_meters: Search radius in meters
|
|
limit: Maximum results
|
|
|
|
Returns:
|
|
List of observations
|
|
"""
|
|
with self._get_connection() as conn:
|
|
with self._get_cursor(conn) as cur:
|
|
cur.execute("""
|
|
SELECT * FROM observations
|
|
WHERE ST_DWithin(
|
|
location::geography,
|
|
ST_SetSRID(ST_MakePoint(%s, %s), 4326)::geography,
|
|
%s
|
|
)
|
|
ORDER BY timestamp DESC
|
|
LIMIT %s
|
|
""", (lng, lat, radius_meters, limit))
|
|
|
|
rows = cur.fetchall()
|
|
return [self._row_to_observation(row) for row in rows]
|
|
|
|
def find_by_defect(
|
|
self,
|
|
defect_code: str,
|
|
min_confidence: float = 0.5,
|
|
limit: int = 100
|
|
) -> List[Observation]:
|
|
"""
|
|
Find observations with specific defect
|
|
|
|
Args:
|
|
defect_code: Defect code
|
|
min_confidence: Minimum confidence threshold
|
|
limit: Maximum results
|
|
|
|
Returns:
|
|
List of observations
|
|
"""
|
|
with self._get_connection() as conn:
|
|
with self._get_cursor(conn) as cur:
|
|
cur.execute("""
|
|
SELECT * FROM observations
|
|
WHERE findings @> '[{"code": "%s"}]'::jsonb
|
|
AND EXISTS (
|
|
SELECT 1 FROM jsonb_array_elements(findings) as f
|
|
WHERE (f->>'code') = %s
|
|
AND (f->>'confidence')::float >= %s
|
|
)
|
|
ORDER BY timestamp DESC
|
|
LIMIT %s
|
|
""", (defect_code, defect_code, min_confidence, limit))
|
|
|
|
rows = cur.fetchall()
|
|
return [self._row_to_observation(row) for row in rows]
|
|
|
|
def _row_to_observation(self, row: Dict) -> Observation:
|
|
"""Convert database row to Observation object"""
|
|
from observation_models import Finding, Media, GeoTag, AIAnalysis, Weather
|
|
|
|
# Parse findings
|
|
findings_data = json.loads(row['findings']) if row['findings'] else []
|
|
findings = [Finding(**f) for f in findings_data]
|
|
|
|
# Parse media
|
|
media_data = json.loads(row['media']) if row['media'] else []
|
|
media = []
|
|
for m in media_data:
|
|
if m.get('geotag'):
|
|
m['geotag'] = GeoTag(**m['geotag'])
|
|
media.append(Media(**m))
|
|
|
|
# Parse weather
|
|
weather = None
|
|
if row['weather']:
|
|
weather = Weather(**json.loads(row['weather']))
|
|
|
|
# Parse AI analysis
|
|
ai_analysis = None
|
|
if row['ai_analysis']:
|
|
ai_analysis = AIAnalysis(**json.loads(row['ai_analysis']))
|
|
|
|
return Observation(
|
|
id=row['id'],
|
|
timestamp=row['timestamp'],
|
|
object_goid=row['object_goid'],
|
|
observer=row['observer'],
|
|
findings=findings,
|
|
media=media,
|
|
weather=weather,
|
|
ai_analysis=ai_analysis
|
|
)
|
|
|
|
def _calculate_trend(self, goid: str) -> Optional[str]:
|
|
"""Calculate trend direction for an object"""
|
|
trend = self.get_trend(goid, months=3)
|
|
|
|
if len(trend.data_points) < 2:
|
|
return None
|
|
|
|
conditions = [dp['condition'] for dp in trend.data_points if dp['condition'] is not None]
|
|
|
|
if len(conditions) < 2:
|
|
return None
|
|
|
|
first = conditions[0]
|
|
last = conditions[-1]
|
|
|
|
if last < first:
|
|
return "improving"
|
|
elif last > first:
|
|
return "degrading"
|
|
else:
|
|
return "stable"
|
|
|
|
def get_degrading_objects(
|
|
self,
|
|
threshold: float = 1.0,
|
|
limit: int = 100
|
|
) -> List[Dict]:
|
|
"""
|
|
Find objects that are degrading over time
|
|
|
|
Args:
|
|
threshold: Minimum condition change to flag
|
|
limit: Maximum results
|
|
|
|
Returns:
|
|
List of degrading objects with details
|
|
"""
|
|
with self._get_connection() as conn:
|
|
with self._get_cursor(conn) as cur:
|
|
cur.execute("""
|
|
WITH condition_changes AS (
|
|
SELECT
|
|
object_goid,
|
|
MIN(timestamp) as first_obs,
|
|
MAX(timestamp) as last_obs,
|
|
MIN((ai_analysis->>'overall_condition')::int) as min_condition,
|
|
MAX((ai_analysis->>'overall_condition')::int) as max_condition,
|
|
COUNT(*) as obs_count
|
|
FROM observations
|
|
WHERE timestamp >= NOW() - INTERVAL '6 months'
|
|
GROUP BY object_goid
|
|
HAVING COUNT(*) >= 2
|
|
)
|
|
SELECT
|
|
object_goid,
|
|
first_obs,
|
|
last_obs,
|
|
min_condition as first_condition,
|
|
max_condition as last_condition,
|
|
(max_condition - min_condition) as condition_change,
|
|
obs_count
|
|
FROM condition_changes
|
|
WHERE (max_condition - min_condition) >= %s
|
|
ORDER BY condition_change DESC
|
|
LIMIT %s
|
|
""", (threshold, limit))
|
|
|
|
return [dict(row) for row in cur.fetchall()]
|
|
|
|
|
|
if __name__ == '__main__':
|
|
# Example usage
|
|
store = ObservationStore()
|
|
|
|
from observation_models import Observation, Finding, Media, GeoTag, AIAnalysis
|
|
from datetime import datetime
|
|
|
|
# Create sample observation
|
|
obs = Observation(
|
|
object_goid="BYG-FAC-WIN-GLA-0001",
|
|
observer="zoomer:anna_k",
|
|
findings=[
|
|
Finding(
|
|
type="dirt_accumulation",
|
|
code="2100",
|
|
description="Smuts på fönster",
|
|
measurement="30% coverage",
|
|
confidence=0.94
|
|
)
|
|
],
|
|
media=[
|
|
Media(
|
|
type="image",
|
|
url="https://storage.quixzoom.com/obs/img_0001.jpg",
|
|
geotag=GeoTag(lat=59.3293, lng=18.0686, accuracy=2.1)
|
|
)
|
|
],
|
|
ai_analysis=AIAnalysis(
|
|
model="infrastructure-v3.2",
|
|
overall_condition=3,
|
|
recommended_action="schedule_cleaning"
|
|
)
|
|
)
|
|
|
|
# Save
|
|
obs_id = store.create(obs)
|
|
print(f"Created observation: {obs_id}")
|
|
|
|
# Retrieve
|
|
retrieved = store.get(obs_id)
|
|
print(f"Retrieved: {retrieved.object_goid if retrieved else 'Not found'}")
|