ADR-011: Four-Layer Data Architecture — Raw Archive → Knowledge → Ontology → Decision
- Layer 1: ArchiveArtifact — immutable original with retention policy - Layer 2: KnowledgeArtifact — extracted knowledge (observations, segmentations, feature vectors, relations) - Layer 3: Knowledge Graph / Ontology (documented, not implemented) - Layer 4: Decision Intelligence (existing DecisionCase) - DataLifecycle: tracks every step with artifact lineage - Key principle: AI models trained on curated datasets, not whole archive - Ontology answers 'what does it mean in our domain?' Long-term goal: Every observation converted once to structured knowledge, reused infinitely for analysis, decisions, training. Next: PR-005A — Minimal Mission Import UI for MVP-0
This commit is contained in:
@@ -0,0 +1,67 @@
|
||||
# ADR-011: Four-Layer Data Architecture
|
||||
|
||||
## Status
|
||||
Accepted
|
||||
|
||||
## Context
|
||||
We need a clear separation between raw data, extracted knowledge, and decision intelligence to avoid building expensive, tightly-coupled AI systems.
|
||||
|
||||
## Decision
|
||||
Separate into four layers:
|
||||
|
||||
### Layer 1: Raw Archive (cheap, immutable)
|
||||
- Original files never changed
|
||||
- Hash guarantees integrity
|
||||
- Storage policy: active (30 days) → archive (long-term, cheaper)
|
||||
|
||||
### Layer 2: Knowledge Extraction
|
||||
- Objects, geometry, GPS, classifications, bounding boxes, embeddings
|
||||
- Observations and Evidence created
|
||||
- Structured knowledge used for most queries, not video files
|
||||
|
||||
### Layer 3: Knowledge Graph / Ontology
|
||||
- Road A12 → has Observation → Crack → belongs to Area → Stockholm
|
||||
- Answers: "Show all main roads where cracking increased last 12 months"
|
||||
- Comes from knowledge graph, not by reading video files
|
||||
|
||||
### Layer 4: Decision Intelligence
|
||||
- Verified Decision Cases
|
||||
- Curated datasets for AI training
|
||||
- Business impact tracking
|
||||
|
||||
## Key Principles
|
||||
|
||||
1. **Archive Artifact** = Original file, never modified
|
||||
2. **Knowledge Artifact** = Extracted knowledge, can be re-generated
|
||||
3. **Ontology before model** — taxonomy answers "what does it mean in our domain?"
|
||||
4. **AI models trained on curated datasets**, not whole archive
|
||||
|
||||
## Data Lifecycle
|
||||
|
||||
```
|
||||
Upload → Immutable Archive → Metadata Extraction → Knowledge Extraction
|
||||
→ Ontology Mapping → Decision Pipeline → Learning → Archive Retention
|
||||
```
|
||||
|
||||
Each step produces a new Artifact with version history.
|
||||
|
||||
## Long-term Goal
|
||||
|
||||
"Every real observation is converted once to structured knowledge and can then be reused infinitely for analysis, decision support, history, training, and future AI models."
|
||||
|
||||
## Consequences
|
||||
|
||||
### Positive
|
||||
- Raw data = revisionable archive
|
||||
- Real product = growing knowledge base + verified Decision Cases
|
||||
- Can swap AI models without losing originals
|
||||
- Cheaper storage (archive vs active)
|
||||
|
||||
### Negative
|
||||
- More complex pipeline
|
||||
- Need to manage extraction models
|
||||
- Knowledge graph maintenance
|
||||
|
||||
## Related
|
||||
- ADR-002: Artifact is common base contract
|
||||
- ADR-003: Event Sourcing for traceability
|
||||
@@ -0,0 +1,68 @@
|
||||
/**
|
||||
* Archive Artifact — Immutable Original
|
||||
*
|
||||
* Principle: Original files never change.
|
||||
* Hash guarantees integrity.
|
||||
* Storage policy: active → archive (cheaper long-term).
|
||||
*
|
||||
* ADR-011: Four-Layer Data Architecture
|
||||
* Layer 1: Raw Archive (this file)
|
||||
*/
|
||||
|
||||
import { ArtifactId } from '../common/ids';
|
||||
import { Hash, StorageUri } from '../common/value-objects';
|
||||
|
||||
export interface ArchiveArtifact {
|
||||
readonly id: ArtifactId;
|
||||
readonly originalName: string;
|
||||
readonly mimeType: string;
|
||||
readonly sizeBytes: number;
|
||||
readonly hash: Hash;
|
||||
readonly storageUri: StorageUri;
|
||||
readonly exif?: {
|
||||
readonly device?: string;
|
||||
readonly gpsLat?: number;
|
||||
readonly gpsLng?: number;
|
||||
readonly timestamp?: Date;
|
||||
readonly iso?: number;
|
||||
readonly exposure?: string;
|
||||
readonly focalLength?: string;
|
||||
};
|
||||
readonly uploadedAt: Date;
|
||||
readonly uploadedBy: string;
|
||||
readonly retentionPolicy: 'active' | 'archive';
|
||||
readonly archiveAfterDate?: Date;
|
||||
}
|
||||
|
||||
export interface CreateArchiveArtifactParams {
|
||||
readonly id: ArtifactId;
|
||||
readonly originalName: string;
|
||||
readonly mimeType: string;
|
||||
readonly sizeBytes: number;
|
||||
readonly hash: Hash;
|
||||
readonly storageUri: StorageUri;
|
||||
readonly exif?: ArchiveArtifact['exif'];
|
||||
readonly uploadedBy: string;
|
||||
readonly retentionPolicy?: 'active' | 'archive';
|
||||
}
|
||||
|
||||
export class ArchiveArtifactFactory {
|
||||
static create(params: CreateArchiveArtifactParams): ArchiveArtifact {
|
||||
return {
|
||||
...params,
|
||||
retentionPolicy: params.retentionPolicy ?? 'active',
|
||||
archiveAfterDate: params.retentionPolicy === 'archive'
|
||||
? undefined
|
||||
: new Date(Date.now() + 30 * 24 * 60 * 60 * 1000), // 30 days default
|
||||
uploadedAt: new Date(),
|
||||
};
|
||||
}
|
||||
|
||||
static archive(artifact: ArchiveArtifact): ArchiveArtifact {
|
||||
return {
|
||||
...artifact,
|
||||
retentionPolicy: 'archive',
|
||||
archiveAfterDate: undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* Data Lifecycle — From Reality to Decision
|
||||
*
|
||||
* ADR-011: Four-Layer Data Architecture
|
||||
*
|
||||
* Flow:
|
||||
* Reality (phone, video, images)
|
||||
* ↓
|
||||
* Raw Archive (immutable)
|
||||
* ↓
|
||||
* Knowledge Extraction
|
||||
* ↓
|
||||
* Knowledge Graph / Ontology
|
||||
* ↓
|
||||
* Decision Intelligence
|
||||
*
|
||||
* Each step produces a new Artifact with its own version history.
|
||||
*/
|
||||
|
||||
import { ArchiveArtifact } from './archive-artifact';
|
||||
import { KnowledgeArtifact } from './knowledge-artifact';
|
||||
|
||||
export interface DataLifecycleStep {
|
||||
readonly step: 'upload' | 'archive' | 'extract_metadata' | 'extract_knowledge' | 'ontology_map' | 'decision' | 'learn' | 'retain';
|
||||
readonly inputArtifactIds: string[];
|
||||
readonly outputArtifactIds: string[];
|
||||
readonly timestamp: Date;
|
||||
readonly processor: string; // Model, human, or system
|
||||
}
|
||||
|
||||
export interface DataLifecycle {
|
||||
readonly archiveArtifact: ArchiveArtifact;
|
||||
readonly knowledgeArtifacts: KnowledgeArtifact[];
|
||||
readonly steps: DataLifecycleStep[];
|
||||
}
|
||||
|
||||
export const DataLifecycleRules = {
|
||||
// Original never changes
|
||||
immutableArchive: (archive: ArchiveArtifact): boolean => {
|
||||
return archive.hash !== undefined && archive.hash.value.length > 0;
|
||||
},
|
||||
|
||||
// Knowledge can be re-extracted without touching archive
|
||||
reextractable: (lifecycle: DataLifecycle): boolean => {
|
||||
return lifecycle.archiveArtifact !== undefined;
|
||||
},
|
||||
|
||||
// Every step produces artifacts
|
||||
artifactAtEveryStep: (lifecycle: DataLifecycle): boolean => {
|
||||
return lifecycle.steps.every(step => step.outputArtifactIds.length > 0);
|
||||
},
|
||||
} as const;
|
||||
@@ -0,0 +1,60 @@
|
||||
/**
|
||||
* Knowledge Artifact — Extracted Knowledge
|
||||
*
|
||||
* Principle: Structured knowledge is the real product.
|
||||
* Raw data becomes knowledge once, then reused infinitely.
|
||||
*
|
||||
* ADR-011: Four-Layer Data Architecture
|
||||
* Layer 2: Knowledge Extraction (this file)
|
||||
*/
|
||||
|
||||
import { ArtifactId, ObservationId } from '../common/ids';
|
||||
import { Hash, StorageUri, GeoLocation } from '../common/value-objects';
|
||||
|
||||
export interface KnowledgeArtifact {
|
||||
readonly id: ArtifactId;
|
||||
readonly sourceArchiveId: ArtifactId; // Links to original
|
||||
readonly type: 'observation' | 'segmentation' | 'classification' | 'feature_vector' | 'relation';
|
||||
readonly extractedAt: Date;
|
||||
readonly extractedBy: string; // Model or human
|
||||
readonly confidence: number; // 0.0 to 1.0
|
||||
readonly data: unknown; // Type-specific data
|
||||
readonly hash: Hash;
|
||||
readonly storageUri: StorageUri;
|
||||
readonly lineage: ArtifactId[];
|
||||
}
|
||||
|
||||
// Specific knowledge types
|
||||
export interface ObservationKnowledge {
|
||||
readonly observationId: ObservationId;
|
||||
readonly description: string;
|
||||
readonly location: GeoLocation;
|
||||
readonly boundingBox?: {
|
||||
readonly x: number;
|
||||
readonly y: number;
|
||||
readonly width: number;
|
||||
readonly height: number;
|
||||
};
|
||||
readonly classifications: string[]; // Taxonomy references
|
||||
readonly qualityScore: number;
|
||||
}
|
||||
|
||||
export interface SegmentationKnowledge {
|
||||
readonly maskUri: string;
|
||||
readonly objectClass: string;
|
||||
readonly pixelCount: number;
|
||||
readonly areaMeters: number;
|
||||
}
|
||||
|
||||
export interface FeatureVectorKnowledge {
|
||||
readonly dimensions: number;
|
||||
readonly vector: number[];
|
||||
readonly modelVersion: string;
|
||||
}
|
||||
|
||||
export interface RelationKnowledge {
|
||||
readonly subjectId: string;
|
||||
readonly predicate: string;
|
||||
readonly objectId: string;
|
||||
readonly confidence: number;
|
||||
}
|
||||
@@ -44,6 +44,27 @@ export {
|
||||
Severity,
|
||||
PriorityValue,
|
||||
} from './common/value-objects';
|
||||
|
||||
// Four-Layer Data Architecture
|
||||
export {
|
||||
ArchiveArtifact,
|
||||
CreateArchiveArtifactParams,
|
||||
ArchiveArtifactFactory,
|
||||
} from './artifacts/archive-artifact';
|
||||
|
||||
export {
|
||||
KnowledgeArtifact,
|
||||
ObservationKnowledge,
|
||||
SegmentationKnowledge,
|
||||
FeatureVectorKnowledge,
|
||||
RelationKnowledge,
|
||||
} from './artifacts/knowledge-artifact';
|
||||
|
||||
export {
|
||||
DataLifecycle,
|
||||
DataLifecycleStep,
|
||||
DataLifecycleRules,
|
||||
} from './artifacts/data-lifecycle';
|
||||
export * from './common/enums';
|
||||
export * from './common/errors';
|
||||
|
||||
|
||||
Reference in New Issue
Block a user