bmc_hub/app/admin/archive_bundle.py

"""Portable, checksummed Project CT archive bundles. Never contacts vTiger."""

from __future__ import annotations

import hashlib
import json
import os
import socket
import tempfile
import zipfile
from datetime import datetime, timezone
from typing import Any, Iterable

from psycopg2.extras import RealDictCursor

from app.core.database import get_db_connection, release_db_connection

BUNDLE_SCHEMA_VERSION = 1
ENTRY_NAMES = ("versions.jsonl", "records.jsonl", "relations.jsonl", "checkpoints.jsonl", "files.jsonl")


def file_sha256(path: str) -> str:
    digest = hashlib.sha256()
    with open(path, "rb") as source:
        for chunk in iter(lambda: source.read(1024 * 1024), b""):
            digest.update(chunk)
    return digest.hexdigest()


def _json_line(value: Any) -> bytes:
    return (json.dumps(value, ensure_ascii=False, sort_keys=True, default=str) + "\n").encode("utf-8")


def _write_cursor_entry(archive: zipfile.ZipFile, name: str, cursor, batch_size: int = 500) -> tuple[int, str]:
    digest = hashlib.sha256()
    count = 0
    with archive.open(name, "w") as target:
        while True:
            rows = cursor.fetchmany(batch_size)
            if not rows:
                break
            for row in rows:
                raw = _json_line(dict(row))
                target.write(raw)
                digest.update(raw)
                count += 1
    return count, digest.hexdigest()


def create_archive_bundle(through_version_id: int, initiated_by: int | None = None) -> tuple[str, dict]:
    fd, path = tempfile.mkstemp(prefix="project-ct-", suffix=".zip")
    os.close(fd)
    conn = get_db_connection()
    manifest = {
        "schema_version": BUNDLE_SCHEMA_VERSION,
        "created_at": datetime.now(timezone.utc).isoformat(),
        "source_instance": socket.gethostname(),
        "through_version_id": through_version_id,
        "entries": {},
        "files": {},
        "contains_files": True,
        "document_policy": "metadata_and_original_bytes",
    }
    try:
        with zipfile.ZipFile(path, "w", compression=zipfile.ZIP_DEFLATED, compresslevel=6) as archive:
            queries = {
                "versions.jsonl": ("SELECT * FROM vtiger_archive_versions WHERE id<=%s ORDER BY id", (through_version_id,)),
                "records.jsonl": ("SELECT * FROM vtiger_archive_records WHERE version_id<=%s ORDER BY module,vtiger_id,revision_no", (through_version_id,)),
                "relations.jsonl": ("SELECT * FROM vtiger_archive_relations WHERE version_id<=%s ORDER BY id", (through_version_id,)),
                "checkpoints.jsonl": (
                    """SELECT module,MAX(source_modified_at) AS last_modified_at,MAX(vtiger_id) AS last_vtiger_id,
                              %s::bigint AS last_successful_version_id,NOW() AS updated_at
                       FROM vtiger_archive_records WHERE version_id<=%s GROUP BY module ORDER BY module""",
                    (through_version_id, through_version_id),
                ),
                "files.jsonl": (
                    """SELECT id,version_id,source_module,document_vtiger_id,resource_vtiger_id,filename,content_type,
                              size_bytes,content_sha256,archived_at
                       FROM vtiger_archive_files WHERE version_id<=%s ORDER BY id""",
                    (through_version_id,),
                ),
            }
            for index, (name, (sql, params)) in enumerate(queries.items()):
                cursor = conn.cursor(name=f"project_ct_export_{index}", cursor_factory=RealDictCursor)
                cursor.itersize = 500
                cursor.execute(sql, params)
                count, digest = _write_cursor_entry(archive, name, cursor)
                cursor.close()
                manifest["entries"][name] = {"count": count, "sha256": digest}
            file_rows = conn.cursor(cursor_factory=RealDictCursor)
            file_rows.execute(
                """SELECT DISTINCT ON (content_sha256) content_sha256,content,size_bytes
                   FROM vtiger_archive_files WHERE version_id<=%s ORDER BY content_sha256,id""",
                (through_version_id,),
            )
            for row in file_rows:
                content = bytes(row["content"])
                digest = hashlib.sha256(content).hexdigest()
                if digest != row["content_sha256"]:
                    raise ValueError(f"Lokal filchecksum er ugyldig: {row['content_sha256']}")
                entry_name = f"files/{digest}"
                archive.writestr(entry_name, content)
                manifest["files"][digest] = {"entry": entry_name, "size": len(content), "sha256": digest}
            file_rows.close()
            archive.writestr("manifest.json", json.dumps(manifest, ensure_ascii=False, sort_keys=True, indent=2, default=str))
        manifest["bundle_sha256"] = file_sha256(path)
        return path, manifest
    except Exception:
        if os.path.exists(path):
            os.unlink(path)
        raise
    finally:
        release_db_connection(conn)


def _verify_entry(archive: zipfile.ZipFile, manifest: dict, name: str) -> None:
    expected = ((manifest.get("entries") or {}).get(name) or {}).get("sha256")
    if not expected:
        raise ValueError(f"Manifest mangler checksum for {name}")
    digest = hashlib.sha256()
    with archive.open(name) as source:
        for chunk in iter(lambda: source.read(1024 * 1024), b""):
            digest.update(chunk)
    if digest.hexdigest() != expected:
        raise ValueError(f"Checksumfejl i {name}")


def _rows(archive: zipfile.ZipFile, name: str) -> Iterable[dict]:
    with archive.open(name) as source:
        for line_number, raw in enumerate(source, 1):
            if not raw.strip():
                continue
            try:
                yield json.loads(raw)
            except json.JSONDecodeError as exc:
                raise ValueError(f"Ugyldig JSON i {name}, linje {line_number}") from exc


def import_archive_bundle(path: str, initiated_by: int | None = None) -> dict:
    bundle_digest = file_sha256(path)
    conn = get_db_connection()
    counts = {"versions": 0, "records": 0, "records_skipped": 0, "relations": 0, "checkpoints": 0}
    try:
        with zipfile.ZipFile(path) as archive:
            try:
                manifest = json.loads(archive.read("manifest.json"))
            except (KeyError, json.JSONDecodeError) as exc:
                raise ValueError("Filen er ikke en gyldig Project CT-arkivpakke") from exc
            if manifest.get("schema_version") != BUNDLE_SCHEMA_VERSION:
                raise ValueError(f"Ikke-understøttet arkivformat: {manifest.get('schema_version')}")
            if manifest.get("contains_files") is not True:
                raise ValueError("Pakken mangler originale vTiger-dokumentfiler")
            for name in ENTRY_NAMES:
                _verify_entry(archive, manifest, name)
            for digest, info in (manifest.get("files") or {}).items():
                entry_name = info.get("entry")
                if not entry_name or entry_name not in archive.namelist():
                    raise ValueError(f"Pakken mangler dokumentfil {digest}")
                actual = hashlib.sha256(archive.read(entry_name)).hexdigest()
                if actual != digest or actual != info.get("sha256"):
                    raise ValueError(f"Checksumfejl i dokumentfil {digest}")

            with conn.cursor(cursor_factory=RealDictCursor) as cursor:
                version_map: dict[int, int] = {}
                for row in _rows(archive, "versions.jsonl"):
                    cursor.execute(
                        """INSERT INTO vtiger_archive_versions
                           (sync_kind,status,started_at,completed_at,source_cutoff,module_counts,warnings,
                            critical_errors,control_report,control_approved_at,raw_export_sha256)
                           VALUES (%s,%s,%s,%s,%s,%s::jsonb,%s::jsonb,%s::jsonb,%s::jsonb,%s,%s) RETURNING id""",
                        (row["sync_kind"], row["status"], row["started_at"], row.get("completed_at"), row["source_cutoff"],
                         json.dumps(row.get("module_counts") or {}), json.dumps(row.get("warnings") or []),
                         json.dumps(row.get("critical_errors") or []), json.dumps(row.get("control_report") or {}),
                         row.get("control_approved_at"), row.get("raw_export_sha256")),
                    )
                    version_map[int(row["id"])] = int(cursor.fetchone()["id"])
                    counts["versions"] += 1

                for row in _rows(archive, "records.jsonl"):
                    cursor.execute(
                        "SELECT id FROM vtiger_archive_records WHERE module=%s AND vtiger_id=%s AND payload_sha256=%s",
                        (row["module"], row["vtiger_id"], row["payload_sha256"]),
                    )
                    if cursor.fetchone():
                        counts["records_skipped"] += 1
                        continue
                    cursor.execute(
                        "SELECT COALESCE(MAX(revision_no),0)+1 AS revision FROM vtiger_archive_records WHERE module=%s AND vtiger_id=%s",
                        (row["module"], row["vtiger_id"]),
                    )
                    revision = int(cursor.fetchone()["revision"])
                    cursor.execute(
                        """INSERT INTO vtiger_archive_records
                           (version_id,module,vtiger_id,revision_no,source_created_at,source_modified_at,is_deleted,payload,payload_sha256,archived_at)
                           VALUES (%s,%s,%s,%s,%s,%s,%s,%s::jsonb,%s,%s)""",
                        (version_map[int(row["version_id"])], row["module"], row["vtiger_id"], revision,
                         row.get("source_created_at"), row.get("source_modified_at"), row.get("is_deleted", False),
                         json.dumps(row["payload"], ensure_ascii=False), row["payload_sha256"], row.get("archived_at")),
                    )
                    counts["records"] += 1

                counts["files"] = 0
                counts["files_skipped"] = 0
                for row in _rows(archive, "files.jsonl"):
                    cursor.execute(
                        """SELECT id FROM vtiger_archive_files
                           WHERE source_module=%s AND document_vtiger_id=%s AND resource_vtiger_id=%s AND content_sha256=%s""",
                        (row.get("source_module") or "Documents", row["document_vtiger_id"], row["resource_vtiger_id"], row["content_sha256"]),
                    )
                    if cursor.fetchone():
                        counts["files_skipped"] += 1
                        continue
                    info = manifest["files"][row["content_sha256"]]
                    content = archive.read(info["entry"])
                    cursor.execute(
                        """INSERT INTO vtiger_archive_files
                           (version_id,source_module,document_vtiger_id,resource_vtiger_id,filename,content_type,size_bytes,
                            content_sha256,content,archived_at)
                           VALUES (%s,%s,%s,%s,%s,%s,%s,%s,%s,%s)""",
                        (version_map[int(row["version_id"])], row.get("source_module") or "Documents", row["document_vtiger_id"], row["resource_vtiger_id"],
                         row["filename"], row.get("content_type"), row["size_bytes"], row["content_sha256"], content,
                         row.get("archived_at")),
                    )
                    counts["files"] += 1

                for row in _rows(archive, "relations.jsonl"):
                    cursor.execute(
                        """INSERT INTO vtiger_archive_relations
                           (version_id,source_module,source_vtiger_id,field_name,target_vtiger_id,target_module)
                           VALUES (%s,%s,%s,%s,%s,%s) ON CONFLICT DO NOTHING""",
                        (version_map[int(row["version_id"])], row["source_module"], row["source_vtiger_id"],
                         row["field_name"], row["target_vtiger_id"], row.get("target_module")),
                    )
                    counts["relations"] += cursor.rowcount

                for row in _rows(archive, "checkpoints.jsonl"):
                    mapped_version = version_map.get(int(row["last_successful_version_id"])) if row.get("last_successful_version_id") else None
                    cursor.execute(
                        """INSERT INTO vtiger_archive_checkpoints
                           (module,last_modified_at,last_vtiger_id,last_successful_version_id,updated_at)
                           VALUES (%s,%s,%s,%s,%s) ON CONFLICT(module) DO UPDATE SET
                           last_modified_at=GREATEST(vtiger_archive_checkpoints.last_modified_at,EXCLUDED.last_modified_at),
                           last_vtiger_id=EXCLUDED.last_vtiger_id,last_successful_version_id=EXCLUDED.last_successful_version_id,
                           updated_at=NOW()""",
                        (row["module"], row.get("last_modified_at"), row.get("last_vtiger_id"), mapped_version, row.get("updated_at")),
                    )
                    counts["checkpoints"] += 1
            conn.commit()
            return {"bundle_sha256": bundle_digest, "manifest": manifest, "counts": counts, "contacted_vtiger": False}
    except Exception:
        conn.rollback()
        raise
    finally:
        release_db_connection(conn)