feat(intelligence): add competitor watch system

- Weekly automated scraping of 10 competitors - Structured data storage (competitors.json) - Validation and alerting scripts - First weekly report (2026-07-11) with 4 critical findings: 1. Premise Data acquired by Culmen (Aug 2025) 2. Streetbees bankruptcy (Aug 2025) 3. Moody's/Cape Analytics integration progressing 4. Sweden Platform Work Act proposed (Dec 2026) - Cron scheduled for Saturdays 07:00 UTC
2026-07-11 08:33:06 +00:00
parent 6de2455917
commit 1d80f13ce5
30 changed files with 33184 additions and 0 deletions
@@ -0,0 +1,58 @@
+#!/bin/bash
+# Alert-skript för kritiska konkurrensförändringar
+# Skickar notis vid viktiga händelser
+
+set -euo pipefail
+
+WORKSPACE="/home/bernt/.openclaw/workspace"
+WATCH_DIR="$WORKSPACE/intelligence/competitor-watch"
+DATE=$(date +%Y-%m-%d)
+
+ALERT_LEVEL="${1:-info}"  # info, warning, critical
+MESSAGE="${2:-}"
+
+# Färger för terminal
+RED='\033[0;31m'
+YELLOW='\033[1;33m'
+GREEN='\033[0;32m'
+NC='\033[0m' # No Color
+
+echo "=== Konkurrens-alert — $DATE ==="
+echo "Nivå: $ALERT_LEVEL"
+echo ""
+
+# Logga alerten
+LOG_FILE="$WATCH_DIR/data/alerts.log"
+echo "$(date -Iseconds) [$ALERT_LEVEL] $MESSAGE" >> "$LOG_FILE"
+
+# Visa alert
+ case "$ALERT_LEVEL" in
+    critical)
+        echo -e "${RED}🚨 KRITISK ALERT${NC}"
+        echo "$MESSAGE"
+        echo ""
+        echo "Åtgärd krävs OMGÅENDE. Meddela Erik."
+        ;;
+    warning)
+        echo -e "${YELLOW}⚠️  VARNING${NC}"
+        echo "$MESSAGE"
+        echo ""
+        echo "Övervaka situationen. Uppdatera vid nästa revision."
+        ;;
+    *)
+        echo -e "${GREEN}ℹ️  INFO${NC}"
+        echo "$MESSAGE"
+        ;;
+esac
+
+# Spara alert till fil för historik
+ALERT_FILE="$WATCH_DIR/data/alerts/${DATE}_${ALERT_LEVEL}.txt"
+mkdir -p "$(dirname "$ALERT_FILE")"
+cat > "$ALERT_FILE" << EOF
+Datum: $(date -Iseconds)
+Nivå: $ALERT_LEVEL
+Meddelande: $MESSAGE
+EOF
+
+echo ""
+echo "Alert loggad: $LOG_FILE"
@@ -0,0 +1,260 @@
+#!/bin/bash
+# Konkurrensbevakning — Skrapningsskript
+# Körs veckovis (lördag 07:00 UTC)
+# Skapad: 2026-07-11
+
+set -euo pipefail
+
+WORKSPACE="/home/bernt/.openclaw/workspace"
+WATCH_DIR="$WORKSPACE/intelligence/competitor-watch"
+DATA_DIR="$WATCH_DIR/data"
+REPORTS_DIR="$WATCH_DIR/reports"
+ARCHIVE_DIR="$WATCH_DIR/archive"
+DATE=$(date +%Y-%m-%d)
+YEAR=$(date +%Y)
+TIMESTAMP=$(date +%Y-%m-%d_%H-%M-%S)
+
+# Skapa nödvändiga kataloger
+mkdir -p "$DATA_DIR/signals/$DATE"
+mkdir -p "$REPORTS_DIR"
+mkdir -p "$ARCHIVE_DIR/$YEAR"
+
+echo "=== Konkurrensbevakning — $DATE ==="
+echo "Start: $(date -Iseconds)"
+echo ""
+
+# ============================================================
+# HJÄLPFUNKTIONER
+# ============================================================
+
+fetch_page() {
+    local url="$1"
+    local output="$2"
+    local timeout="${3:-30}"
+    
+    if command -v curl >/dev/null 2>&1; then
+        curl -sL --max-time "$timeout" \
+             -H "User-Agent: Mozilla/5.0 (compatible; LandvexBot/1.0)" \
+             "$url" > "$output" 2>/dev/null || echo "<!-- FETCH_FAILED: $url -->" > "$output"
+    else
+        echo "<!-- CURL_NOT_AVAILABLE -->" > "$output"
+    fi
+}
+
+extract_text() {
+    local file="$1"
+    # Enkel textextraktion — tar bort HTML-taggar
+    sed 's/<[^>]*>//g' "$file" 2>/dev/null | tr -s ' \n' | head -c 50000
+}
+
+# ============================================================
+# 1. SKRAPA KONKURRENTWEBBPLATSER (grundläggande)
+# ============================================================
+
+echo "--- 1. Skrapar konkurrentwebbplatser ---"
+
+# Definiera URL:er att bevaka
+declare -A COMPETITOR_URLS
+COMPETITOR_URLS[we golook]="https://we golook.com"
+COMPETITOR_URLS[premise]="https://www.premise.com"
+COMPETITOR_URLS[nearmap]="https://www.nearmap.com"
+COMPETITOR_URLS[cape]="https://www.capeanalytics.com"
+COMPETITOR_URLS[hivemapper]="https://hivemapper.com"
+COMPETITOR_URLS[vialytics]="https://vialytics.com"
+COMPETITOR_URLS[cyclomedia]="https://www.cyclomedia.com"
+COMPETITOR_URLS[zeitview]="https://www.zeitview.com"
+COMPETITOR_URLS[roamler]="https://www.roamler.com"
+COMPETITOR_URLS[esri]="https://www.esri.com"
+
+# Skrapa varje sida
+for name in "${!COMPETITOR_URLS[@]}"; do
+    url="${COMPETITOR_URLS[$name]}"
+    output="$DATA_DIR/signals/$DATE/${name}.html"
+    
+    echo "  Skrapar: $name ($url)"
+    fetch_page "$url" "$output"
+    
+    # Spara textversion för enkelare diff
+    extract_text "$output" > "$DATA_DIR/signals/$DATE/${name}.txt" 2>/dev/null || true
+    
+    sleep 2  # Respektfull rate limiting
+done
+
+echo ""
+
+# ============================================================
+# 2. SÖK EFTER NYHETER (via web_search om tillgängligt)
+# ============================================================
+
+echo "--- 2. Söker efter nyheter ---"
+
+# Söktermer att bevaka
+SEARCH_TERMS=(
+    "WeGoLook OR Crawford claims"
+    "Premise Data funding OR acquisition"
+    "Nearmap insurance OR pricing"
+    "Cape Analytics Moody's"
+    "Hivemapper OR Bee Maps"
+    "vialytics kommuner"
+    "EU Platform Work Directive 2026"
+    "crowdsourced data collection market"
+    "property intelligence AI insurance"
+    "drone inspection pricing"
+)
+
+# Spara söktermer för manuell uppföljning
+echo "${SEARCH_TERMS[@]}" | tr ' ' '\n' > "$DATA_DIR/signals/$DATE/search_terms.txt"
+
+echo "  Söktermer sparade: ${#SEARCH_TERMS[@]} termer"
+echo "  (Manuell uppföljning via web_search krävs)"
+echo ""
+
+# ============================================================
+# 3. KONTROLLERA FÖRÄNDRINGAR (diff mot föregående vecka)
+# ============================================================
+
+echo "--- 3. Jämför med föregående vecka ---"
+
+PREVIOUS_WEEK=$(find "$DATA_DIR/signals" -maxdepth 1 -type d -name "2026-*" | sort | tail -2 | head -1)
+CURRENT_WEEK="$DATA_DIR/signals/$DATE"
+
+if [ -n "$PREVIOUS_WEEK" ] && [ "$PREVIOUS_WEEK" != "$CURRENT_WEEK" ]; then
+    echo "  Föregående vecka: $(basename $PREVIOUS_WEEK)"
+    
+    # För varje fil i nuvarande vecka, jämför med föregående
+    for file in "$CURRENT_WEEK"/*.txt; do
+        if [ -f "$file" ]; then
+            filename=$(basename "$file")
+            prev_file="$PREVIOUS_WEEK/$filename"
+            
+            if [ -f "$prev_file" ]; then
+                # Gör diff
+                diff_output=$(diff -u "$prev_file" "$file" 2>/dev/null | head -100 || true)
+                if [ -n "$diff_output" ]; then
+                    echo "  🆕 FÖRÄNDRING: $filename"
+                    echo "$diff_output" > "$DATA_DIR/signals/$DATE/diff_${filename%.txt}.txt"
+                fi
+            fi
+        fi
+    done
+else
+    echo "  Ingen föregående vecka att jämföra med (första körningen)"
+fi
+
+echo ""
+
+# ============================================================
+# 4. GENERERA RAPPORT
+# ============================================================
+
+echo "--- 4. Genererar rapport ---"
+
+REPORT_FILE="$REPORTS_DIR/${DATE}-weekly-report.md"
+
+# Hitta diff-filer
+DIFF_FILES=$(find "$DATA_DIR/signals/$DATE" -name "diff_*.txt" 2>/dev/null | wc -l)
+
+cat > "$REPORT_FILE" << EOF
+# Veckorapport — Konkurrensbevakning
+**Datum:** $DATE  
+**Revision:** Vecka $(date +%V)  
+**Nästa revision:** $(date -d '+7 days' +%Y-%m-%d)
+
+---
+
+## Sammanfattning
+
+| Mått | Värde |
+|------|-------|
+| Konkurrenter bevakade | ${#COMPETITOR_URLS[@]} |
+| Webbsidor skrapade | ${#COMPETITOR_URLS[@]} |
+| Förändringar identifierade | $DIFF_FILES |
+| Status | ✅ Genomförd |
+
+---
+
+## Förändringar denna vecka
+
+EOF
+
+if [ "$DIFF_FILES" -gt 0 ]; then
+    echo "Följande förändringar identifierades:" >> "$REPORT_FILE"
+    echo "" >> "$REPORT_FILE"
+    
+    for diff_file in "$DATA_DIR/signals/$DATE"/diff_*.txt; do
+        if [ -f "$diff_file" ]; then
+            competitor=$(basename "$diff_file" | sed 's/diff_//;s/\.txt//')
+            echo "### $competitor" >> "$REPORT_FILE"
+            echo '```diff' >> "$REPORT_FILE"
+            cat "$diff_file" >> "$REPORT_FILE"
+            echo '```' >> "$REPORT_FILE"
+            echo "" >> "$REPORT_FILE"
+        fi
+    done
+else
+    echo "Inga förändringar identifierade denna vecka." >> "$REPORT_FILE"
+    echo "" >> "$REPORT_FILE"
+fi
+
+cat >> "$REPORT_FILE" << EOF
+
+---
+
+## Manuella kontroller krävs
+
+- [ ] Kontrollera Crunchbase för kapitalrundor
+- [ ] Kontrollera LinkedIn för nyanställningar (signal om expansion)
+- [ ] Kontrollera Google Alerts för nyheter
+- [ ] Kontrollera prissidor för ändringar
+- [ ] Kontrollera EU-regleringar (Platform Work Directive)
+
+---
+
+## Rekommenderade åtgärder
+
+*(Fylls i efter manuell granskning)*
+
+---
+
+## Rådata
+
+- HTML-filer: \`data/signals/$DATE/\`
+- Textextraktion: \`data/signals/$DATE/*.txt\`
+- Diff-filer: \`data/signals/$DATE/diff_*.txt\`
+
+---
+
+*Rapport genererad automatiskt: $(date -Iseconds)*
+EOF
+
+# Uppdatera "latest"-länk
+ln -sf "$REPORT_FILE" "$REPORTS_DIR/latest.md"
+
+echo "  Rapport sparad: $REPORT_FILE"
+echo ""
+
+# ============================================================
+# 5. ARKIVERING
+# ============================================================
+
+echo "--- 5. Arkivering ---"
+
+# Komprimera signaler
+tar -czf "$ARCHIVE_DIR/$YEAR/signals-${DATE}.tar.gz" -C "$DATA_DIR/signals" "$DATE" 2>/dev/null || true
+
+echo "  Signaler arkiverade"
+echo ""
+
+# ============================================================
+# SLUT
+# ============================================================
+
+echo "=== Slutfört ==="
+echo "Slut: $(date -Iseconds)"
+echo "Rapport: $REPORT_FILE"
+echo ""
+echo "Nästa steg:"
+echo "  1. Granska rapporten manuellt"
+echo "  2. Utför web_search på söktermerna"
+echo "  3. Uppdatera competitors.json vid behov"
+echo "  4. Meddela Erik om kritiska förändringar"
@@ -0,0 +1,142 @@
+#!/bin/bash
+# Valideringsskript för konkurrensdata
+# Kontrollerar att all data är komplett och konsekvent
+
+set -euo pipefail
+
+WORKSPACE="/home/bernt/.openclaw/workspace"
+WATCH_DIR="$WORKSPACE/intelligence/competitor-watch"
+DATA_DIR="$WATCH_DIR/data"
+DATE=$(date +%Y-%m-%d)
+
+ERRORS=0
+WARNINGS=0
+
+echo "=== Validering av konkurrensdata — $DATE ==="
+echo ""
+
+# 1. Kontrollera att competitors.json finns och är giltig JSON
+echo "--- 1. Kontrollerar competitors.json ---"
+if [ ! -f "$DATA_DIR/competitors.json" ]; then
+    echo "  ❌ ERROR: competitors.json saknas"
+    ERRORS=$((ERRORS + 1))
+else
+    if python3 -c "import json; json.load(open('$DATA_DIR/competitors.json'))" 2>/dev/null; then
+        echo "  ✅ competitors.json är giltig JSON"
+        
+        # Kontrollera antal konkurrenter
+        COUNT=$(python3 -c "import json; d=json.load(open('$DATA_DIR/competitors.json')); print(len(d['competitors']))")
+        echo "  📊 Antal konkurrenter: $COUNT"
+        
+        if [ "$COUNT" -lt 10 ]; then
+            echo "  ⚠️  WARNING: Färre än 10 konkurrenter ($COUNT)"
+            WARNINGS=$((WARNINGS + 1))
+        fi
+    else
+        echo "  ❌ ERROR: competitors.json är ogiltig JSON"
+        ERRORS=$((ERRORS + 1))
+    fi
+fi
+echo ""
+
+# 2. Kontrollera att alla konkurrenter har nödvändiga fält
+echo "--- 2. Kontrollerar obligatoriska fält ---"
+REQUIRED_FIELDS=("rank" "name" "segment" "priority" "status" "threat_level")
+
+for field in "${REQUIRED_FIELDS[@]}"; do
+    MISSING=$(python3 -c "
+import json
+d=json.load(open('$DATA_DIR/competitors.json'))
+missing = [c['name'] for c in d['competitors'] if '$field' not in c or not c['$field']]
+print('\n'.join(missing)) if missing else print('OK')
+" 2>/dev/null)
+    
+    if [ "$MISSING" = "OK" ]; then
+        echo "  ✅ $field: alla konkurrenter har värdet"
+    else
+        echo "  ⚠️  WARNING: $field saknas för:"
+        echo "$MISSING" | sed 's/^/     - /'
+        WARNINGS=$((WARNINGS + 1))
+    fi
+done
+echo ""
+
+# 3. Kontrollera att prisdata är aktuell
+echo "--- 3. Kontrollerar prisdata ---"
+STALE_PRICES=$(python3 -c "
+import json
+from datetime import datetime, timedelta
+d=json.load(open('$DATA_DIR/competitors.json'))
+stale = []
+for c in d['competitors']:
+    if 'pricing' in c and 'last_verified' in c['pricing']:
+        lv = c['pricing']['last_verified']
+        if lv:
+            date = datetime.strptime(lv, '%Y-%m-%d')
+            if datetime.now() - date > timedelta(days=90):
+                stale.append(f\"{c['name']} (senast: {lv})\")
+print('\n'.join(stale)) if stale else print('OK')
+" 2>/dev/null)
+
+if [ "$STALE_PRICES" = "OK" ]; then
+    echo "  ✅ All prisdata är aktuell (< 90 dagar)"
+else
+    echo "  ⚠️  WARNING: Föråldrade priser:"
+    echo "$STALE_PRICES" | sed 's/^/     - /'
+    WARNINGS=$((WARNINGS + 1))
+fi
+echo ""
+
+# 4. Kontrollera att senaste rapporten finns
+echo "--- 4. Kontrollerar rapporter ---"
+LATEST_REPORT=$(find "$WATCH_DIR/reports" -name "*-weekly-report.md" | sort | tail -1)
+if [ -n "$LATEST_REPORT" ]; then
+    REPORT_DATE=$(basename "$LATEST_REPORT" | cut -d'-' -f1-3)
+    echo "  ✅ Senaste rapport: $REPORT_DATE"
+    
+    # Kontrollera om rapporten är äldre än 8 dagar
+    REPORT_AGE=$(( ($(date +%s) - $(date -d "$REPORT_DATE" +%s)) / 86400 ))
+    if [ "$REPORT_AGE" -gt 8 ]; then
+        echo "  ⚠️  WARNING: Rapporten är $REPORT_AGE dagar gammal"
+        WARNINGS=$((WARNINGS + 1))
+    fi
+else
+    echo "  ⚠️  WARNING: Ingen rapport hittad"
+    WARNINGS=$((WARNINGS + 1))
+fi
+echo ""
+
+# 5. Kontrollera att skrapade filer finns
+echo "--- 5. Kontrollerar skrapade data ---"
+LATEST_SIGNALS=$(find "$DATA_DIR/signals" -maxdepth 1 -type d -name "2026-*" | sort | tail -1)
+if [ -n "$LATEST_SIGNALS" ]; then
+    SIGNAL_COUNT=$(find "$LATEST_SIGNALS" -type f | wc -l)
+    echo "  ✅ Senaste skrapning: $(basename $LATEST_SIGNALS) ($SIGNAL_COUNT filer)"
+    
+    SIGNAL_AGE=$(( ($(date +%s) - $(date -d "$(basename $LATEST_SIGNALS)" +%s)) / 86400 ))
+    if [ "$SIGNAL_AGE" -gt 8 ]; then
+        echo "  ⚠️  WARNING: Skrapningen är $SIGNAL_AGE dagar gammal"
+        WARNINGS=$((WARNINGS + 1))
+    fi
+else
+    echo "  ⚠️  WARNING: Inga skrapade signaler hittade"
+    WARNINGS=$((WARNINGS + 1))
+fi
+echo ""
+
+# Sammanfattning
+echo "=== Validering slutförd ==="
+echo "Errors: $ERRORS"
+echo "Warnings: $WARNINGS"
+echo ""
+
+if [ "$ERRORS" -gt 0 ]; then
+    echo "❌ VALIDERING MISSLYCKADES — åtgärda errors"
+    exit 1
+elif [ "$WARNINGS" -gt 0 ]; then
+    echo "⚠️  VALIDERING GODKÄND med varningar"
+    exit 0
+else
+    echo "✅ VALIDERING GODKÄND"
+    exit 0
+fi