landvex: Datafabrik + Vision + Infrastruktur för 50GB skalning
Datafabrik: - Skördare: crawler, källvitlista, upphandlingsskördare - Extraktor: LLM-baserad schemastyrd extraktion - Upplösare: Entitetsupplösning och deduplicering - Köer: Schemalagd / kunddriven / fält - Agentorkestrering: 20+ parallella agenter Vision: - Identify-modell: ResNet50 + kontrastivt lärande - Träningspipeline: NT-Xent loss - Vektordatabas: FAISS för snabb sökning - OCR-pipeline: Typskyltsläsning Infrastruktur: - Docker Compose production - Terraform för AWS ECS - Prometheus + Grafana monitorering - Neo4j + FAISS + MinIO + Redis
This commit is contained in:
Bernt
@@ -0,0 +1,72 @@
|
||||
{
|
||||
"dashboard": {
|
||||
"title": "Landvex IOL",
|
||||
"tags": ["landvex", "iol"],
|
||||
"timezone": "UTC",
|
||||
"panels": [
|
||||
{
|
||||
"title": "API Requests/min",
|
||||
"type": "graph",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "rate(landvex_api_requests_total[5m])",
|
||||
"legendFormat": "{{method}} {{endpoint}}"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 0, "y": 0}
|
||||
},
|
||||
{
|
||||
"title": "Identify Konfidens",
|
||||
"type": "graph",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "histogram_quantile(0.95, rate(landvex_identify_confidence_bucket[5m]))",
|
||||
"legendFormat": "p95"
|
||||
},
|
||||
{
|
||||
"expr": "histogram_quantile(0.50, rate(landvex_identify_confidence_bucket[5m]))",
|
||||
"legendFormat": "p50"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 0}
|
||||
},
|
||||
{
|
||||
"title": "Graf-storlek",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "landvex_graph_nodes_total",
|
||||
"legendFormat": "Noder"
|
||||
},
|
||||
{
|
||||
"expr": "landvex_graph_edges_total",
|
||||
"legendFormat": "Kanter"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 4, "w": 6, "x": 0, "y": 8}
|
||||
},
|
||||
{
|
||||
"title": "Kö-storlek",
|
||||
"type": "stat",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "landvex_queue_size",
|
||||
"legendFormat": "{{queue}}"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 4, "w": 6, "x": 6, "y": 8}
|
||||
},
|
||||
{
|
||||
"title": "Kostnad per faktum",
|
||||
"type": "graph",
|
||||
"targets": [
|
||||
{
|
||||
"expr": "landvex_cost_per_fact_usd",
|
||||
"legendFormat": "USD/faktum"
|
||||
}
|
||||
],
|
||||
"gridPos": {"h": 8, "w": 12, "x": 12, "y": 8}
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,23 @@
|
||||
global:
|
||||
scrape_interval: 15s
|
||||
evaluation_interval: 15s
|
||||
|
||||
scrape_configs:
|
||||
- job_name: 'landvex-api'
|
||||
static_configs:
|
||||
- targets: ['api:8080']
|
||||
metrics_path: '/metrics'
|
||||
|
||||
- job_name: 'landvex-datafabrik'
|
||||
static_configs:
|
||||
- targets: ['datafabrik:8080']
|
||||
metrics_path: '/metrics'
|
||||
|
||||
- job_name: 'neo4j'
|
||||
static_configs:
|
||||
- targets: ['neo4j:7474']
|
||||
metrics_path: '/metrics'
|
||||
|
||||
- job_name: 'prometheus'
|
||||
static_configs:
|
||||
- targets: ['localhost:9090']
|
||||
@@ -0,0 +1,162 @@
|
||||
# Landvex Infrastruktur — AWS Terraform
|
||||
# Skalar till 50GB+ agentkraft
|
||||
|
||||
terraform {
|
||||
required_providers {
|
||||
aws = {
|
||||
source = "hashicorp/aws"
|
||||
version = "~> 5.0"
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
provider "aws" {
|
||||
region = var.aws_region
|
||||
}
|
||||
|
||||
# ─── VPC och nätverk ─────────────────────────────────────
|
||||
resource "aws_vpc" "landvex" {
|
||||
cidr_block = "10.0.0.0/16"
|
||||
enable_dns_hostnames = true
|
||||
enable_dns_support = true
|
||||
|
||||
tags = {
|
||||
Name = "landvex-vpc"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_subnet" "public" {
|
||||
count = 2
|
||||
vpc_id = aws_vpc.landvex.id
|
||||
cidr_block = "10.0.${count.index + 1}.0/24"
|
||||
availability_zone = data.aws_availability_zones.available.names[count.index]
|
||||
map_public_ip_on_launch = true
|
||||
|
||||
tags = {
|
||||
Name = "landvex-public-${count.index + 1}"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_subnet" "private" {
|
||||
count = 2
|
||||
vpc_id = aws_vpc.landvex.id
|
||||
cidr_block = "10.0.${count.index + 10}.0/24"
|
||||
availability_zone = data.aws_availability_zones.available.names[count.index]
|
||||
|
||||
tags = {
|
||||
Name = "landvex-private-${count.index + 1}"
|
||||
}
|
||||
}
|
||||
|
||||
data "aws_availability_zones" "available" {
|
||||
state = "available"
|
||||
}
|
||||
|
||||
# ─── ECR Repositories ────────────────────────────────────
|
||||
resource "aws_ecr_repository" "api" {
|
||||
name = "landvex-api"
|
||||
image_tag_mutability = "MUTABLE"
|
||||
|
||||
image_scanning_configuration {
|
||||
scan_on_push = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_ecr_repository" "datafabrik" {
|
||||
name = "landvex-datafabrik"
|
||||
image_tag_mutability = "MUTABLE"
|
||||
|
||||
image_scanning_configuration {
|
||||
scan_on_push = true
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_ecr_repository" "vision" {
|
||||
name = "landvex-vision"
|
||||
image_tag_mutability = "MUTABLE"
|
||||
|
||||
image_scanning_configuration {
|
||||
scan_on_push = true
|
||||
}
|
||||
}
|
||||
|
||||
# ─── ECS Cluster ─────────────────────────────────────────
|
||||
resource "aws_ecs_cluster" "landvex" {
|
||||
name = "landvex-cluster"
|
||||
|
||||
setting {
|
||||
name = "containerInsights"
|
||||
value = "enabled"
|
||||
}
|
||||
}
|
||||
|
||||
resource "aws_ecs_cluster_capacity_providers" "landvex" {
|
||||
cluster_name = aws_ecs_cluster.landvex.name
|
||||
|
||||
capacity_providers = ["FARGATE", "FARGATE_SPOT"]
|
||||
|
||||
default_capacity_provider_strategy {
|
||||
base = 1
|
||||
weight = 1
|
||||
capacity_provider = "FARGATE"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── S3 Bucket för datalagring ───────────────────────────
|
||||
resource "aws_s3_bucket" "landvex_data" {
|
||||
bucket = "landvex-data-${var.environment}"
|
||||
}
|
||||
|
||||
resource "aws_s3_bucket_versioning" "landvex_data" {
|
||||
bucket = aws_s3_bucket.landvex_data.id
|
||||
versioning_configuration {
|
||||
status = "Enabled"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── RDS (PostgreSQL för metadata) ───────────────────────
|
||||
resource "aws_db_instance" "landvex_metadata" {
|
||||
identifier = "landvex-metadata"
|
||||
allocated_storage = 100
|
||||
max_allocated_storage = 1000
|
||||
engine = "postgres"
|
||||
engine_version = "16"
|
||||
instance_class = "db.r6g.xlarge"
|
||||
db_name = "landvex"
|
||||
username = "landvex"
|
||||
password = var.db_password
|
||||
skip_final_snapshot = true
|
||||
|
||||
tags = {
|
||||
Name = "landvex-metadata"
|
||||
}
|
||||
}
|
||||
|
||||
# ─── Variables ───────────────────────────────────────────
|
||||
variable "aws_region" {
|
||||
description = "AWS region"
|
||||
default = "eu-north-1"
|
||||
}
|
||||
|
||||
variable "environment" {
|
||||
description = "Environment (dev/staging/prod)"
|
||||
default = "prod"
|
||||
}
|
||||
|
||||
variable "db_password" {
|
||||
description = "Database password"
|
||||
sensitive = true
|
||||
}
|
||||
|
||||
# ─── Outputs ─────────────────────────────────────────────
|
||||
output "vpc_id" {
|
||||
value = aws_vpc.landvex.id
|
||||
}
|
||||
|
||||
output "ecr_api_url" {
|
||||
value = aws_ecr_repository.api.repository_url
|
||||
}
|
||||
|
||||
output "s3_bucket" {
|
||||
value = aws_s3_bucket.landvex_data.bucket
|
||||
}
|
||||
Reference in New Issue
Block a user