#!/usr/bin/env python3 """Compare local regenerated model inputs with committed inputs. This script is intentionally read-only with respect to data/. It writes a report under _local/reports/ and exits nonzero when files differ or are missing. """ from __future__ import annotations import csv import hashlib import os from pathlib import Path from typing import Iterable MODEL_INPUTS = [ "text_data.csv", "expert.csv", "lr_data.csv", "union_mapping.csv", "party_families.csv", ] def normalized_rows(path: Path) -> Iterable[list[str]]: with path.open(newline="", encoding="utf-8") as fh: reader = csv.reader(fh) for row in reader: yield ["" if value is None else value.strip() for value in row] def normalized_hash(path: Path) -> str: h = hashlib.sha256() for row in normalized_rows(path): h.update(("\x1f".join(row) + "\n").encode("utf-8")) return h.hexdigest() def csv_shape(path: Path) -> tuple[int, int, list[str]]: with path.open(newline="", encoding="utf-8") as fh: reader = csv.reader(fh) try: header = next(reader) except StopIteration: return 0, 0, [] rows = sum(1 for _ in reader) return rows, len(header), header def first_difference(left: Path, right: Path, max_scan: int = 100000) -> str: for idx, (lrow, rrow) in enumerate(zip(normalized_rows(left), normalized_rows(right)), start=1): if lrow != rrow: return f"line {idx}: committed={lrow[:8]!r} generated={rrow[:8]!r}" if idx >= max_scan: break return "no differing row found in scan window; row counts may differ" def main() -> int: repo_root = Path(__file__).resolve().parents[1] committed_dir = Path(os.environ.get("PARTY2D_COMMITTED_INPUT_DIR", repo_root / "data")) generated_dir = Path(os.environ.get("PARTY2D_GENERATED_INPUT_DIR", repo_root / "_local" / "generated-inputs")) report_dir = Path(os.environ.get("PARTY2D_REPORT_DIR", repo_root / "_local" / "reports")) report_dir.mkdir(parents=True, exist_ok=True) report_path = report_dir / "input_comparison.md" lines: list[str] = [] lines.append("# Generated input comparison") lines.append("") lines.append(f"Committed input dir: `{committed_dir}`") lines.append(f"Generated input dir: `{generated_dir}`") lines.append("") lines.append("| file | status | committed rows | generated rows | committed hash | generated hash | notes |") lines.append("| --- | --- | ---: | ---: | --- | --- | --- |") ok = True for name in MODEL_INPUTS: committed = committed_dir / name generated = generated_dir / name if not committed.exists() or not generated.exists(): ok = False lines.append( f"| `{name}` | missing | | | | | committed exists={committed.exists()}, generated exists={generated.exists()} |" ) continue c_rows, c_cols, c_header = csv_shape(committed) g_rows, g_cols, g_header = csv_shape(generated) c_hash = normalized_hash(committed) g_hash = normalized_hash(generated) same = c_hash == g_hash status = "match" if same else "diff" if not same: ok = False notes = [] if c_cols != g_cols: notes.append(f"columns {c_cols}!={g_cols}") if c_header != g_header: notes.append("header differs") if c_rows != g_rows: notes.append(f"rows {c_rows}!={g_rows}") if not same and not notes: notes.append(first_difference(committed, generated)) lines.append( f"| `{name}` | {status} | {c_rows} | {g_rows} | `{c_hash[:12]}` | `{g_hash[:12]}` | {'; '.join(notes)} |" ) lines.append("") lines.append("Committed inputs were not modified by this comparison.") report_path.write_text("\n".join(lines) + "\n", encoding="utf-8") print(f"Wrote comparison report: {report_path}") if ok: print("Generated inputs match committed inputs.") return 0 print("Generated inputs differ from committed inputs or are missing.") return 1 if __name__ == "__main__": raise SystemExit(main())