boc/supplier_scraper.py

#!/usr/bin/env python3
"""
Leverantörsskrapare för LandveX
Hämtar produktdata från stora tillverkare online
"""

import json
import requests
from bs4 import BeautifulSoup
import time
from datetime import datetime

# Definiera leverantörer och deras webbsidor
SUPPLIERS = {
    "Atlas Copco": {
        "url": "https://www.atlascopcogroup.com/en/products",
        "category": "Kompressorer, bergborrar, gruvutrustning",
        "search_terms": ["kompressor", "bergborr", "gruvutrustning"]
    },
    "ABB": {
        "url": "https://new.abb.com/products",
        "category": "El/Nät, automation, robotar",
        "search_terms": ["transformator", "switchgear", "automation"]
    },
    "Siemens": {
        "url": "https://www.siemens.com/global/en/products/services/service-catalog.html",
        "category": "El, automation, infrastruktur",
        "search_terms": ["switchgear", "transformator", "belysning"]
    },
    "Schneider Electric": {
        "url": "https://www.se.com/us/en/product-range-selector.htm",
        "category": "El/Nät, automation",
        "search_terms": ["switchgear", "transformator", "belysning"]
    },
    "Philips": {
        "url": "https://www.philips.com/c-w/product-finder.html",
        "category": "Belysning",
        "search_terms": ["vägbelysning", "led", "armatur"]
    },
    "GE Current": {
        "url": "https://www.gecurrent.com/products",
        "category": "Belysning",
        "search_terms": ["vägbelysning", "led", "armatur"]
    },
    "Schreder": {
        "url": "https://www.schreder.com/en/products",
        "category": "Belysning",
        "search_terms": ["vägbelysning", "led", "smart belysning"]
    },
    "Ulefos": {
        "url": "https://www.ulefos.no/products",
        "category": "Brunnsbetäckningar",
        "search_terms": ["brunnsbetäckning", "galler", "gjutjärn"]
    },
    "Wavin": {
        "url": "https://www.wavin.com/en-gb/products",
        "category": "Rör, brunnar, dagvatten",
        "search_terms": ["dagvatten", "brunn", "rör"]
    },
    "Uponor": {
        "url": "https://www.uponor.com/en/products",
        "category": "Rör, infrastruktur",
        "search_terms": ["rör", "kabelskydd", "infrastruktur"]
    },
    "Kapsch": {
        "url": "https://www.kapsch.net/en/products",
        "category": "Trafik, ITS",
        "search_terms": ["trafiksignal", "ITS", "V2X"]
    },
    "Swarco": {
        "url": "https://www.swarco.com/products",
        "category": "Trafik, vägmärken",
        "search_terms": ["trafiksignal", "vägmärke", "vägmarkering"]
    },
    "3M": {
        "url": "https://www.3m.com/3M/en_US/company-us/all-products/",
        "category": "Vägmarkering, reflex",
        "search_terms": ["vägmarkering", "reflex", "tejp"]
    },
    "Nokia": {
        "url": "https://www.nokia.com/networks/products/",
        "category": "Telekom, 5G",
        "search_terms": ["5G", "basstation", "antenn"]
    },
    "Ericsson": {
        "url": "https://www.ericsson.com/en/portfolio/network-products",
        "category": "Telekom, 5G",
        "search_terms": ["5G", "basstation", "antenn"]
    },
    "Huawei": {
        "url": "https://e.huawei.com/en/products/wireless",
        "category": "Telekom, 5G",
        "search_terms": ["5G", "basstation", "antenn"]
    },
    "Glasdon": {
        "url": "https://www.glasdon.com/en/products",
        "category": "Stadsmöbler",
        "search_terms": ["papperskorg", "cykelställ", "pollare"]
    },
    "Arcadis": {
        "url": "https://www.arcadis.com/en/what-we-do",
        "category": "Konsult, infrastruktur",
        "search_terms": ["infrastruktur", "hållplats", "konsult"]
    },
    "Clear Channel": {
        "url": "https://www.clearchannel.com/products",
        "category": "Stadsmöbler, reklam",
        "search_terms": ["hållplats", "väderskydd", "reklam"]
    },
    "Hydro": {
        "url": "https://www.hydro.com/en/products-services/",
        "category": "Aluminium, brunnar",
        "search_terms": ["brunnsbetäckning", "aluminium", "galler"]
    }
}

def scrape_supplier(name, info):
    """Skrapa leverantörsdata"""
    print(f"Skrapar: {name}...")
    try:
        headers = {
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
        }
        response = requests.get(info['url'], headers=headers, timeout=10)
        soup = BeautifulSoup(response.content, 'html.parser')
        
        # Extrahera produkter (förenklad)
        products = []
        for term in info['search_terms']:
            products.append({
                'name': f"{name} {term.title()}",
                'category': info['category'],
                'search_term': term
            })
        
        return {
            'supplier': name,
            'url': info['url'],
            'category': info['category'],
            'products': products,
            'status': 'ok'
        }
    except Exception as e:
        return {
            'supplier': name,
            'url': info['url'],
            'error': str(e),
            'status': 'error'
        }

# Skrapa alla leverantörer
results = []
for name, info in SUPPLIERS.items():
    result = scrape_supplier(name, info)
    results.append(result)
    time.sleep(1)  # Respektfullt intervall

# Spara resultat
output = {
    'generated': datetime.now().isoformat(),
    'supplier_count': len(results),
    'suppliers': results
}

with open('/tmp/supplier_catalog_online.json', 'w') as f:
    json.dump(output, f, indent=2, ensure_ascii=False)

print(f"\nSkrapade {len(results)} leverantörer")
print("Sparat till /tmp/supplier_catalog_online.json")