boc/intelligence/competitor-watch/scripts/scrape.sh

#!/bin/bash
# Konkurrensbevakning — Skrapningsskript
# Körs veckovis (lördag 07:00 UTC)
# Skapad: 2026-07-11

set -euo pipefail

WORKSPACE="/home/bernt/.openclaw/workspace"
WATCH_DIR="$WORKSPACE/intelligence/competitor-watch"
DATA_DIR="$WATCH_DIR/data"
REPORTS_DIR="$WATCH_DIR/reports"
ARCHIVE_DIR="$WATCH_DIR/archive"
DATE=$(date +%Y-%m-%d)
YEAR=$(date +%Y)
TIMESTAMP=$(date +%Y-%m-%d_%H-%M-%S)

# Skapa nödvändiga kataloger
mkdir -p "$DATA_DIR/signals/$DATE"
mkdir -p "$REPORTS_DIR"
mkdir -p "$ARCHIVE_DIR/$YEAR"

echo "=== Konkurrensbevakning — $DATE ==="
echo "Start: $(date -Iseconds)"
echo ""

# ============================================================
# HJÄLPFUNKTIONER
# ============================================================

fetch_page() {
    local url="$1"
    local output="$2"
    local timeout="${3:-30}"
    
    if command -v curl >/dev/null 2>&1; then
        curl -sL --max-time "$timeout" \
             -H "User-Agent: Mozilla/5.0 (compatible; LandvexBot/1.0)" \
             "$url" > "$output" 2>/dev/null || echo "<!-- FETCH_FAILED: $url -->" > "$output"
    else
        echo "<!-- CURL_NOT_AVAILABLE -->" > "$output"
    fi
}

extract_text() {
    local file="$1"
    # Enkel textextraktion — tar bort HTML-taggar
    sed 's/<[^>]*>//g' "$file" 2>/dev/null | tr -s ' \n' | head -c 50000
}

# ============================================================
# 1. SKRAPA KONKURRENTWEBBPLATSER (grundläggande)
# ============================================================

echo "--- 1. Skrapar konkurrentwebbplatser ---"

# Definiera URL:er att bevaka
declare -A COMPETITOR_URLS
COMPETITOR_URLS[we golook]="https://we golook.com"
COMPETITOR_URLS[premise]="https://www.premise.com"
COMPETITOR_URLS[nearmap]="https://www.nearmap.com"
COMPETITOR_URLS[cape]="https://www.capeanalytics.com"
COMPETITOR_URLS[hivemapper]="https://hivemapper.com"
COMPETITOR_URLS[vialytics]="https://vialytics.com"
COMPETITOR_URLS[cyclomedia]="https://www.cyclomedia.com"
COMPETITOR_URLS[zeitview]="https://www.zeitview.com"
COMPETITOR_URLS[roamler]="https://www.roamler.com"
COMPETITOR_URLS[esri]="https://www.esri.com"

# Skrapa varje sida
for name in "${!COMPETITOR_URLS[@]}"; do
    url="${COMPETITOR_URLS[$name]}"
    output="$DATA_DIR/signals/$DATE/${name}.html"
    
    echo "  Skrapar: $name ($url)"
    fetch_page "$url" "$output"
    
    # Spara textversion för enkelare diff
    extract_text "$output" > "$DATA_DIR/signals/$DATE/${name}.txt" 2>/dev/null || true
    
    sleep 2  # Respektfull rate limiting
done

echo ""

# ============================================================
# 2. SÖK EFTER NYHETER (via web_search om tillgängligt)
# ============================================================

echo "--- 2. Söker efter nyheter ---"

# Söktermer att bevaka
SEARCH_TERMS=(
    "WeGoLook OR Crawford claims"
    "Premise Data funding OR acquisition"
    "Nearmap insurance OR pricing"
    "Cape Analytics Moody's"
    "Hivemapper OR Bee Maps"
    "vialytics kommuner"
    "EU Platform Work Directive 2026"
    "crowdsourced data collection market"
    "property intelligence AI insurance"
    "drone inspection pricing"
)

# Spara söktermer för manuell uppföljning
echo "${SEARCH_TERMS[@]}" | tr ' ' '\n' > "$DATA_DIR/signals/$DATE/search_terms.txt"

echo "  Söktermer sparade: ${#SEARCH_TERMS[@]} termer"
echo "  (Manuell uppföljning via web_search krävs)"
echo ""

# ============================================================
# 3. KONTROLLERA FÖRÄNDRINGAR (diff mot föregående vecka)
# ============================================================

echo "--- 3. Jämför med föregående vecka ---"

PREVIOUS_WEEK=$(find "$DATA_DIR/signals" -maxdepth 1 -type d -name "2026-*" | sort | tail -2 | head -1)
CURRENT_WEEK="$DATA_DIR/signals/$DATE"

if [ -n "$PREVIOUS_WEEK" ] && [ "$PREVIOUS_WEEK" != "$CURRENT_WEEK" ]; then
    echo "  Föregående vecka: $(basename $PREVIOUS_WEEK)"
    
    # För varje fil i nuvarande vecka, jämför med föregående
    for file in "$CURRENT_WEEK"/*.txt; do
        if [ -f "$file" ]; then
            filename=$(basename "$file")
            prev_file="$PREVIOUS_WEEK/$filename"
            
            if [ -f "$prev_file" ]; then
                # Gör diff
                diff_output=$(diff -u "$prev_file" "$file" 2>/dev/null | head -100 || true)
                if [ -n "$diff_output" ]; then
                    echo "  🆕 FÖRÄNDRING: $filename"
                    echo "$diff_output" > "$DATA_DIR/signals/$DATE/diff_${filename%.txt}.txt"
                fi
            fi
        fi
    done
else
    echo "  Ingen föregående vecka att jämföra med (första körningen)"
fi

echo ""

# ============================================================
# 4. GENERERA RAPPORT
# ============================================================

echo "--- 4. Genererar rapport ---"

REPORT_FILE="$REPORTS_DIR/${DATE}-weekly-report.md"

# Hitta diff-filer
DIFF_FILES=$(find "$DATA_DIR/signals/$DATE" -name "diff_*.txt" 2>/dev/null | wc -l)

cat > "$REPORT_FILE" << EOF
# Veckorapport — Konkurrensbevakning
**Datum:** $DATE  
**Revision:** Vecka $(date +%V)  
**Nästa revision:** $(date -d '+7 days' +%Y-%m-%d)

---

## Sammanfattning

| Mått | Värde |
|------|-------|
| Konkurrenter bevakade | ${#COMPETITOR_URLS[@]} |
| Webbsidor skrapade | ${#COMPETITOR_URLS[@]} |
| Förändringar identifierade | $DIFF_FILES |
| Status | ✅ Genomförd |

---

## Förändringar denna vecka

EOF

if [ "$DIFF_FILES" -gt 0 ]; then
    echo "Följande förändringar identifierades:" >> "$REPORT_FILE"
    echo "" >> "$REPORT_FILE"
    
    for diff_file in "$DATA_DIR/signals/$DATE"/diff_*.txt; do
        if [ -f "$diff_file" ]; then
            competitor=$(basename "$diff_file" | sed 's/diff_//;s/\.txt//')
            echo "### $competitor" >> "$REPORT_FILE"
            echo '```diff' >> "$REPORT_FILE"
            cat "$diff_file" >> "$REPORT_FILE"
            echo '```' >> "$REPORT_FILE"
            echo "" >> "$REPORT_FILE"
        fi
    done
else
    echo "Inga förändringar identifierade denna vecka." >> "$REPORT_FILE"
    echo "" >> "$REPORT_FILE"
fi

cat >> "$REPORT_FILE" << EOF

---

## Manuella kontroller krävs

- [ ] Kontrollera Crunchbase för kapitalrundor
- [ ] Kontrollera LinkedIn för nyanställningar (signal om expansion)
- [ ] Kontrollera Google Alerts för nyheter
- [ ] Kontrollera prissidor för ändringar
- [ ] Kontrollera EU-regleringar (Platform Work Directive)

---

## Rekommenderade åtgärder

*(Fylls i efter manuell granskning)*

---

## Rådata

- HTML-filer: \`data/signals/$DATE/\`
- Textextraktion: \`data/signals/$DATE/*.txt\`
- Diff-filer: \`data/signals/$DATE/diff_*.txt\`

---

*Rapport genererad automatiskt: $(date -Iseconds)*
EOF

# Uppdatera "latest"-länk
ln -sf "$REPORT_FILE" "$REPORTS_DIR/latest.md"

echo "  Rapport sparad: $REPORT_FILE"
echo ""

# ============================================================
# 5. ARKIVERING
# ============================================================

echo "--- 5. Arkivering ---"

# Komprimera signaler
tar -czf "$ARCHIVE_DIR/$YEAR/signals-${DATE}.tar.gz" -C "$DATA_DIR/signals" "$DATE" 2>/dev/null || true

echo "  Signaler arkiverade"
echo ""

# ============================================================
# SLUT
# ============================================================

echo "=== Slutfört ==="
echo "Slut: $(date -Iseconds)"
echo "Rapport: $REPORT_FILE"
echo ""
echo "Nästa steg:"
echo "  1. Granska rapporten manuellt"
echo "  2. Utför web_search på söktermerna"
echo "  3. Uppdatera competitors.json vid behov"
echo "  4. Meddela Erik om kritiska förändringar"