Bernt
7d40cf95bf
- FastAPI Identify API (port 8081) - Neo4j propertygraf (64 noder, 80 kanter) - Ingest-pipeline (Python) - Byggscript + integrationstester - Docker Compose setup
234 lines
12 KiB
Python
234 lines
12 KiB
Python
#!/usr/bin/env python3
|
|
# Landvex Ingest v0 — deterministisk pipeline
|
|
# kandidatposter (extraktorns output) -> resolve -> merge -> validera -> master/graf/bounty-förslag
|
|
import json, re, difflib
|
|
from pathlib import Path
|
|
|
|
OUT = Path("/mnt/user-data/outputs")
|
|
IDAG = "2026-07-04"
|
|
|
|
PREDIKAT = {"monteras_pa","star_pa","ansluts_med","matas_fran","ingar_i","del_av",
|
|
"ersatter","ersatts_av","kompatibel_med","forvaxlas_med","tillverkas_av","foljer_standard","variant_av"}
|
|
VERIF_RANK = {"obekraftad":0,"kallbelagd":1,"faltverifierad":2,"tillverkarbekraftad":3}
|
|
DOMANER = {"TRP","ELN","VAT","TEL","BYG","PRK","JVG","HMN","FLG"}
|
|
ID_RE = re.compile(r"^LVX-(TRP|ELN|VAT|TEL|BYG|PRK|JVG|HMN|FLG)-[0-9]{4}$|^LVX-P-[0-9]{7}$")
|
|
|
|
def load(p):
|
|
with open(OUT/p, encoding="utf-8") as f: return json.load(f)
|
|
|
|
def save(p,obj):
|
|
with open(OUT/p,"w",encoding="utf-8") as f: json.dump(obj,f,ensure_ascii=False,indent=2)
|
|
|
|
def norm_bet(b):
|
|
b = b.upper().split(":")[0]
|
|
b = re.sub(r"\((KARTLÄGGS|KARTLAGGS)\)","",b)
|
|
return re.sub(r"\s+"," ",b).strip()
|
|
|
|
TOM = ("", None)
|
|
def ar_tom(v): return v in TOM or v == [] or v == {}
|
|
|
|
class Ingest:
|
|
def __init__(self):
|
|
self.poster = {p["lvx_id"]: p for p in load("lvx-klassposter-master-v2.json")["poster"]}
|
|
self.modeller = {}
|
|
for fn in ["lvx-produktmodeller-vag2.json","lvx-produktmodeller-vag3.json"]:
|
|
for m in load(fn)["poster"]: self.modeller[m["lvx_id"]] = m
|
|
self.reg = load("lvx-id-register.json")
|
|
self.nya_modeller, self.logg, self.konflikter = [], [], []
|
|
self.stat = {"uppdaterade":0,"nya":0,"nivalyft":0,"kallor":0,"standarder":0}
|
|
|
|
# ---- resolve ----
|
|
def hitta(self, kand):
|
|
rid = kand.get("lvx_id")
|
|
if rid:
|
|
if rid in self.poster: return self.poster[rid]
|
|
if rid in self.modeller: return self.modeller[rid]
|
|
raise KeyError(f"{kand['kandidat_id']}: okänt lvx_id {rid}")
|
|
# entitetsupplösning utan ID: slug, annars fuzzy på namn
|
|
post = kand.get("post",{})
|
|
alla = {**self.poster, **self.modeller}
|
|
for r in alla.values():
|
|
if r.get("slug") and r["slug"] == post.get("slug"): return r
|
|
namn = post.get("namn",{}).get("sv","")
|
|
kandnamn = {r["namn"]["sv"]: r for r in alla.values()}
|
|
tr = difflib.get_close_matches(namn, kandnamn.keys(), n=1, cutoff=0.88)
|
|
return kandnamn[tr[0]] if tr else None
|
|
|
|
# ---- merge-primitiver ----
|
|
def merge_kallor(self, rec, nya):
|
|
kl = rec.setdefault("proveniens",{}).setdefault("kallor",[])
|
|
urls = {k.get("url") for k in kl}
|
|
for k in nya:
|
|
if k.get("url") not in urls:
|
|
kl.append(k); urls.add(k.get("url"))
|
|
self.stat["kallor"] += 1
|
|
self.logg.append(f"{rec['lvx_id']}: +källa {k.get('url','?')[:60]}")
|
|
|
|
def merge_standarder(self, rec, nya):
|
|
st = rec.setdefault("standarder",[])
|
|
idx = {norm_bet(s["beteckning"]): i for i,s in enumerate(st) if s.get("beteckning")}
|
|
for s in nya:
|
|
n = norm_bet(s["beteckning"])
|
|
if n in idx:
|
|
st[idx[n]] = s
|
|
self.logg.append(f"{rec['lvx_id']}: standard uppdaterad {s['beteckning']}")
|
|
else:
|
|
st.append(s)
|
|
self.logg.append(f"{rec['lvx_id']}: +standard {s['beteckning']}")
|
|
self.stat["standarder"] += 1
|
|
|
|
def fyll(self, base, patch, rid, stig=""):
|
|
for k,v in patch.items():
|
|
here = f"{stig}.{k}" if stig else k
|
|
if k not in base or ar_tom(base.get(k)):
|
|
base[k] = v
|
|
self.logg.append(f"{rid}: fyllde {here}")
|
|
elif isinstance(base[k], dict) and isinstance(v, dict):
|
|
self.fyll(base[k], v, rid, here)
|
|
elif base[k] != v:
|
|
self.konflikter.append(f"{rid}: {here} — befintligt värde behålls (granskningskö i produktion)")
|
|
|
|
# ---- åtgärder ----
|
|
def uppdatera(self, kand):
|
|
rec = self.hitta(kand)
|
|
patch = dict(kand.get("patch",{}))
|
|
if "standarder" in patch: self.merge_standarder(rec, patch.pop("standarder"))
|
|
prov = patch.pop("proveniens", {})
|
|
if prov.get("kallor"): self.merge_kallor(rec, prov["kallor"])
|
|
if patch: self.fyll(rec, patch, rec["lvx_id"])
|
|
fs = kand.get("verifieringsniva_forslag")
|
|
if fs:
|
|
nu = rec["proveniens"].get("verifieringsniva","obekraftad")
|
|
if VERIF_RANK[fs] > VERIF_RANK[nu]:
|
|
if VERIF_RANK[fs] >= 1 and not rec["proveniens"].get("kallor"):
|
|
self.konflikter.append(f"{rec['lvx_id']}: nivålyft nekad — inga källor")
|
|
else:
|
|
rec["proveniens"]["verifieringsniva"] = fs
|
|
rec["proveniens"]["konfidens"] = max(rec["proveniens"].get("konfidens",0), 0.72)
|
|
self.stat["nivalyft"] += 1
|
|
self.logg.append(f"{rec['lvx_id']}: {nu} → {fs}")
|
|
rec["proveniens"]["uppdaterad"] = IDAG
|
|
self.stat["uppdaterade"] += 1
|
|
|
|
def ny(self, kand):
|
|
if self.hitta(kand):
|
|
self.konflikter.append(f"{kand['kandidat_id']}: dublett — matchar befintlig post, konverteras ej")
|
|
return
|
|
post = kand["post"]
|
|
typ = kand.get("serie",{}).get("typ","modell" if post.get("posttyp")=="produktmodell" else "klass")
|
|
if typ == "modell":
|
|
post["lvx_id"] = f"LVX-P-{self.reg['P_nasta']:07d}"
|
|
self.reg["P_nasta"] += 1
|
|
if not post.get("parent"): raise ValueError(f"{kand['kandidat_id']}: modell utan parent")
|
|
self.modeller[post["lvx_id"]] = post
|
|
self.nya_modeller.append(post)
|
|
else:
|
|
dom, tier = post["doman"], str(kand["serie"]["tier"])
|
|
nn = self.reg["klass_nasta"].setdefault(dom,{}).setdefault(tier,1)
|
|
post["lvx_id"] = f"LVX-{dom}-0{tier}{nn:02d}"
|
|
self.reg["klass_nasta"][dom][tier] = nn + 1
|
|
self.poster[post["lvx_id"]] = post
|
|
post.setdefault("proveniens",{}).setdefault("skapad",IDAG)
|
|
post["proveniens"]["uppdaterad"] = IDAG
|
|
self.stat["nya"] += 1
|
|
self.logg.append(f"NY: {post['lvx_id']} — {post['namn']['sv']}")
|
|
|
|
def kor(self, fil):
|
|
for kand in load(fil)["kandidater"]:
|
|
(self.uppdatera if kand["atgard"]=="uppdatera" else self.ny)(kand)
|
|
|
|
# ---- validering (samma regler som konsolidering v2) ----
|
|
def validera(poster, modeller):
|
|
fel = []
|
|
def kolla(rec, typ):
|
|
rid = rec.get("lvx_id","?")
|
|
if not ID_RE.match(rid): fel.append(f"{rid}: ID-format")
|
|
if rec.get("doman") not in DOMANER: fel.append(f"{rid}: domän")
|
|
if not (rec.get("namn",{}).get("sv") and rec["namn"].get("en")): fel.append(f"{rid}: namn")
|
|
prov = rec.get("proveniens",{})
|
|
if prov.get("verifieringsniva") not in VERIF_RANK: fel.append(f"{rid}: verifieringsnivå")
|
|
for pr in rec.get("problem",[]):
|
|
if not pr.get("kalla"): fel.append(f"{rid}: problem utan källa")
|
|
for rel in rec.get("relationer",[]):
|
|
if rel.get("predikat") not in PREDIKAT: fel.append(f"{rid}: predikat {rel.get('predikat')}")
|
|
if typ=="modell" and not rec.get("parent"): fel.append(f"{rid}: parent")
|
|
if prov.get("verifieringsniva")=="kallbelagd" and not prov.get("kallor"): fel.append(f"{rid}: källbelagd utan källor")
|
|
for p in poster.values(): kolla(p,"klass")
|
|
for m in modeller.values(): kolla(m,"modell")
|
|
return fel
|
|
|
|
# ---- graf ----
|
|
def bygg_graf(poster, modeller):
|
|
kanter, olosta = [], {}
|
|
def multi(m): return " / " in m and all(ID_RE.match(x.strip()) for x in m.split(" / "))
|
|
def kant(s,pred,obj,niva,beskr=""):
|
|
typ = "lvx" if ID_RE.match(obj or "") else ("standard" if pred=="foljer_standard" else "text")
|
|
if typ=="text": olosta[obj] = olosta.get(obj,0)+1
|
|
kanter.append({"subjekt":s,"predikat":pred,"objekt":obj,"objekt_typ":typ,"niva":niva,
|
|
**({"beskrivning":beskr} if beskr else {})})
|
|
def emit(s,pred,m,niva,beskr=""):
|
|
if multi(m):
|
|
for x in m.split(" / "): kant(s,pred,x.strip(),niva,beskr)
|
|
else: kant(s,pred,m,niva,beskr)
|
|
for p in poster.values():
|
|
for rel in p.get("relationer",[]): emit(p["lvx_id"],rel["predikat"],rel["mal"],"klass",rel.get("beskrivning",""))
|
|
for st in p.get("standarder",[]):
|
|
if st.get("beteckning"): kant(p["lvx_id"],"foljer_standard",st["beteckning"],"klass",st.get("roll",""))
|
|
for m in modeller.values():
|
|
kant(m["lvx_id"],"tillhor_klass",m["parent"],"modell")
|
|
for e in m.get("historik",{}).get("ersatter",[]): kant(m["lvx_id"],"ersatter",e,"modell")
|
|
for e in m.get("historik",{}).get("ersatts_av",[]): kant(m["lvx_id"],"ersatts_av",e,"modell")
|
|
for rel in m.get("relationer",[]): emit(m["lvx_id"],rel["predikat"],rel["mal"],"modell",rel.get("beskrivning",""))
|
|
return kanter, olosta
|
|
|
|
# ---- flyhjulet: bounty-förslag ur nya/uppgraderade poster ----
|
|
def bounty_forslag(poster):
|
|
try: tackta = {b["mal_lvx"] for b in load("lvx-zoomer-bounties-vag1.json")["bounties"]}
|
|
except FileNotFoundError: tackta = set()
|
|
ut = []
|
|
for p in poster.values():
|
|
if VERIF_RANK[p["proveniens"]["verifieringsniva"]] < 1: continue
|
|
falt = p.get("ai",{}).get("ocr",{}).get("falt",[])
|
|
if falt and p["lvx_id"] not in tackta:
|
|
ut.append({"forslag_id": f"BNTY-K-{len(ut)+1:03d}", "mal_lvx": p["lvx_id"],
|
|
"titel": f"OCR-fält: {p['namn']['sv']}", "ocr_falt": falt,
|
|
"grund": "auto-genererad ur postens OCR-fält (ingest-pipelinen)"})
|
|
return ut
|
|
|
|
if __name__ == "__main__":
|
|
ing = Ingest()
|
|
ing.kor("intag/kandidatposter-vag7.json")
|
|
fel = validera(ing.poster, ing.modeller)
|
|
kanter, olosta = bygg_graf(ing.poster, ing.modeller)
|
|
vf = {}
|
|
for p in ing.poster.values():
|
|
v = p["proveniens"]["verifieringsniva"]; vf[v] = vf.get(v,0)+1
|
|
|
|
save("lvx-klassposter-master-v3.json", {
|
|
"beskrivning":"Landvex IOL — master v3 (ingest-pipelinen, våg 7). Ersätter v2.",
|
|
"genererad": IDAG, "antal_poster": len(ing.poster), "verifieringsfordelning": vf,
|
|
"poster":[ing.poster[k] for k in sorted(ing.poster)]})
|
|
if ing.nya_modeller:
|
|
save("lvx-produktmodeller-auto.json", {
|
|
"beskrivning":"Modellposter mintade av ingest-pipelinen.", "genererad": IDAG, "poster": ing.nya_modeller})
|
|
lvx_k = sum(1 for k in kanter if k["objekt_typ"]=="lvx"); std_k = sum(1 for k in kanter if k["objekt_typ"]=="standard")
|
|
save("lvx-graf-kanter-v3.json", {
|
|
"beskrivning":"Landvex IOL — grafexport v3 (efter ingest våg 7).", "genererad": IDAG,
|
|
"noder":{"objektklasser":len(ing.poster),"produktmodeller":len(ing.modeller)},
|
|
"statistik":{"kanter_totalt":len(kanter),"upplosta_lvx_kanter":lvx_k,"standardkanter":std_k,"textnoder_olosta":len(olosta)},
|
|
"olosta_platshallare":[{"text":k,"forekomster":v} for k,v in sorted(olosta.items())],
|
|
"kanter":kanter})
|
|
bf = bounty_forslag(ing.poster)
|
|
save("lvx-bounty-kandidater-vag7.json", {"beskrivning":"Auto-genererade bounty-förslag (granskas innan publicering).","genererad":IDAG,"forslag":bf})
|
|
ing.reg["uppdaterad"] = IDAG
|
|
save("lvx-id-register.json", ing.reg)
|
|
|
|
print(f"Uppdaterade poster: {ing.stat['uppdaterade']} | Nya: {ing.stat['nya']} (register P_nasta={ing.reg['P_nasta']})")
|
|
print(f"Nivålyft: {ing.stat['nivalyft']} | +källor: {ing.stat['kallor']} | standarder: {ing.stat['standarder']}")
|
|
print(f"Verifieringsfördelning: {vf}")
|
|
print(f"Modeller totalt: {len(ing.modeller)} | Kanter: {len(kanter)} ({lvx_k} lvx / {std_k} standard / {len(olosta)} textnoder)")
|
|
print(f"Bounty-förslag: {len(bf)} | Konflikter: {len(ing.konflikter)} | Valideringsfel: {len(fel)}")
|
|
for r in ing.logg: print(" ·", r)
|
|
for k in ing.konflikter: print(" !", k)
|
|
for f_ in fel: print(" !!", f_)
|