#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
valider_kilder.py
Automatisk validerings- og audit-script for Danske Aabne Datakilder.
Pinger alle endpoints, maaler svartid, verificerer HTTP-status og opdaterer/genererer
baade CSV-kataloget og verifikationsloggen med fuld provenance.
"""

import csv
import json
import os
import ssl
import sys
import time
import urllib.request
import urllib.error
from datetime import datetime, timezone

SOURCES_JSON_PATH = os.path.join(os.path.dirname(__file__), "danske_aabne_datakilder.json")
CATALOG_CSV_PATH = os.path.join(os.path.dirname(__file__), "danske_aabne_datakilder.csv")
AUDIT_LOG_PATH = os.path.join(os.path.dirname(__file__), "audit_log.jsonl")

# Standard SSL context (tolerant over for interne cert-kaeder)
ssl_ctx = ssl.create_default_context()
ssl_ctx.check_hostname = False
ssl_ctx.verify_mode = ssl.CERT_NONE

DEFAULT_USER_AGENT = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) DanishOpenDataAuditor/1.0"


def ping_endpoint(url, timeout=7):
    """Pinger en URL og returnerer (status_code, response_ms, error_msg)."""
    headers = {"User-Agent": DEFAULT_USER_AGENT}
    req = urllib.request.Request(url, headers=headers)
    start_time = time.time()
    try:
        with urllib.request.urlopen(req, timeout=timeout, context=ssl_ctx) as response:
            latency_ms = int((time.time() - start_time) * 1000)
            return response.status, latency_ms, ""
    except urllib.error.HTTPError as e:
        latency_ms = int((time.time() - start_time) * 1000)
        return e.code, latency_ms, f"HTTPError: {e.code}"
    except urllib.error.URLError as e:
        latency_ms = int((time.time() - start_time) * 1000)
        return 0, latency_ms, f"URLError: {str(e.reason)[:80]}"
    except Exception as e:
        latency_ms = int((time.time() - start_time) * 1000)
        return 0, latency_ms, f"{type(e).__name__}: {str(e)[:80]}"


def export_csv(sources, output_path):
    """Eksporterer den flade kildeliste til en ren UTF-8 CSV fil."""
    fields = [
        "id",
        "titel",
        "organisation",
        "kategori",
        "adgangstype",
        "formater",
        "licens",
        "url_portal",
        "url_api_eller_data",
        "licens_url",
        "opdateringsfrekvens",
        "verifikationsstatus",
        "sidst_verificeret",
        "beskrivelse"
    ]
    with open(output_path, "w", encoding="utf-8", newline="") as f:
        writer = csv.DictWriter(f, fieldnames=fields, delimiter=",", quoting=csv.QUOTE_MINIMAL)
        writer.writeheader()
        for item in sources:
            row = {
                "id": item.get("id", ""),
                "titel": item.get("titel", ""),
                "organisation": item.get("organisation", ""),
                "kategori": item.get("kategori", ""),
                "adgangstype": item.get("adgangstype", ""),
                "formater": ";".join(item.get("formater", [])) if isinstance(item.get("formater"), list) else str(item.get("formater", "")),
                "licens": item.get("licens", ""),
                "url_portal": item.get("url_portal", ""),
                "url_api_eller_data": item.get("url_api_eller_data", ""),
                "licens_url": item.get("licens_url", ""),
                "opdateringsfrekvens": item.get("opdateringsfrekvens", ""),
                "verifikationsstatus": item.get("verifikationsstatus", ""),
                "sidst_verificeret": item.get("sidst_verificeret", ""),
                "beskrivelse": item.get("beskrivelse", "").replace("\n", " ").strip()
            }
            writer.writerow(row)


def main():
    if not os.path.exists(SOURCES_JSON_PATH):
        print(f"Fejl: Kunne ikke finde datakataloget paa {SOURCES_JSON_PATH}")
        sys.exit(1)

    with open(SOURCES_JSON_PATH, "r", encoding="utf-8") as f:
        catalog_data = json.load(f)

    sources = catalog_data.get("kilder", [])
    now_iso = datetime.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
    today_str = datetime.now(timezone.utc).strftime("%Y-%m-%d")

    print(f"=== Starter kontrol og validering af {len(sources)} kilder [{now_iso}] ===\n")
    audit_records = []
    success_count = 0

    for idx, s in enumerate(sources, 1):
        target_url = s.get("url_api_eller_data") or s.get("url_portal")
        title = s.get("titel", "Uden titel")
        print(f"[{idx:02d}/{len(sources):02d}] Tjekker {title[:40]:<40} ... ", end="", flush=True)

        status_code, latency_ms, err = ping_endpoint(target_url)
        is_ok = (200 <= status_code < 400)

        status_label = "OK" if is_ok else f"FEJL ({status_code or err})"
        print(f"{status_label} ({latency_ms} ms)")

        if is_ok:
            success_count += 1
            s["verifikationsstatus"] = "Aktiv / Verificeret"
            s["sidst_verificeret"] = today_str
        else:
            # Hvis det er særlige API'er med login/whitelist (fx CVR), noteres det
            if status_code in (401, 403):
                s["verifikationsstatus"] = f"Aktiv (Kraever adgang/token, HTTP {status_code})"
                s["sidst_verificeret"] = today_str
                success_count += 1
            else:
                s["verifikationsstatus"] = f"Advarsel: {err or status_code}"

        audit_records.append({
            "timestamp": now_iso,
            "id": s.get("id"),
            "url": target_url,
            "http_status": status_code,
            "latency_ms": latency_ms,
            "error": err
        })

    # Opdater katalog metadata
    catalog_data["metadata"]["sidst_verificeret"] = today_str
    catalog_data["metadata"]["antal_kilder"] = len(sources)

    # Gem opdateret JSON
    with open(SOURCES_JSON_PATH, "w", encoding="utf-8") as f:
        json.dump(catalog_data, f, ensure_ascii=False, indent=2)

    # Opdater CSV eksport
    export_csv(sources, CATALOG_CSV_PATH)

    # Append til audit_log.jsonl
    with open(AUDIT_LOG_PATH, "a", encoding="utf-8") as f:
        for rec in audit_records:
            f.write(json.dumps(rec, ensure_ascii=False) + "\n")

    print(f"\nValidering fuldfoert: {success_count}/{len(sources)} kilder svarede tilfredsstillende.")
    print(f"Opdateret: {SOURCES_JSON_PATH}")
    print(f"Eksporteret: {CATALOG_CSV_PATH}")
    print(f"Logget audit til: {AUDIT_LOG_PATH}")


if __name__ == "__main__":
    main()