aseimel
122 lines
4.2 KiB
Python
122 lines
4.2 KiB
Python
#!/usr/bin/env python3
|
|
"""Compare local regenerated model inputs with committed inputs.
|
|
|
|
This script is intentionally read-only with respect to data/. It writes a report
|
|
under _local/reports/ and exits nonzero when files differ or are missing.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import csv
|
|
import hashlib
|
|
import os
|
|
from pathlib import Path
|
|
from typing import Iterable
|
|
|
|
|
|
MODEL_INPUTS = [
|
|
"text_data.csv",
|
|
"expert.csv",
|
|
"lr_data.csv",
|
|
"union_mapping.csv",
|
|
"party_families.csv",
|
|
]
|
|
|
|
|
|
def normalized_rows(path: Path) -> Iterable[list[str]]:
|
|
with path.open(newline="", encoding="utf-8") as fh:
|
|
reader = csv.reader(fh)
|
|
for row in reader:
|
|
yield ["" if value is None else value.strip() for value in row]
|
|
|
|
|
|
def normalized_hash(path: Path) -> str:
|
|
h = hashlib.sha256()
|
|
for row in normalized_rows(path):
|
|
h.update(("\x1f".join(row) + "\n").encode("utf-8"))
|
|
return h.hexdigest()
|
|
|
|
|
|
def csv_shape(path: Path) -> tuple[int, int, list[str]]:
|
|
with path.open(newline="", encoding="utf-8") as fh:
|
|
reader = csv.reader(fh)
|
|
try:
|
|
header = next(reader)
|
|
except StopIteration:
|
|
return 0, 0, []
|
|
rows = sum(1 for _ in reader)
|
|
return rows, len(header), header
|
|
|
|
|
|
def first_difference(left: Path, right: Path, max_scan: int = 100000) -> str:
|
|
for idx, (lrow, rrow) in enumerate(zip(normalized_rows(left), normalized_rows(right)), start=1):
|
|
if lrow != rrow:
|
|
return f"line {idx}: committed={lrow[:8]!r} generated={rrow[:8]!r}"
|
|
if idx >= max_scan:
|
|
break
|
|
return "no differing row found in scan window; row counts may differ"
|
|
|
|
|
|
def main() -> int:
|
|
repo_root = Path(__file__).resolve().parents[1]
|
|
committed_dir = Path(os.environ.get("PARTY2D_COMMITTED_INPUT_DIR", repo_root / "data"))
|
|
generated_dir = Path(os.environ.get("PARTY2D_GENERATED_INPUT_DIR", repo_root / "_local" / "generated-inputs"))
|
|
report_dir = Path(os.environ.get("PARTY2D_REPORT_DIR", repo_root / "_local" / "reports"))
|
|
report_dir.mkdir(parents=True, exist_ok=True)
|
|
report_path = report_dir / "input_comparison.md"
|
|
|
|
lines: list[str] = []
|
|
lines.append("# Generated input comparison")
|
|
lines.append("")
|
|
lines.append(f"Committed input dir: `{committed_dir}`")
|
|
lines.append(f"Generated input dir: `{generated_dir}`")
|
|
lines.append("")
|
|
lines.append("| file | status | committed rows | generated rows | committed hash | generated hash | notes |")
|
|
lines.append("| --- | --- | ---: | ---: | --- | --- | --- |")
|
|
|
|
ok = True
|
|
for name in MODEL_INPUTS:
|
|
committed = committed_dir / name
|
|
generated = generated_dir / name
|
|
if not committed.exists() or not generated.exists():
|
|
ok = False
|
|
lines.append(
|
|
f"| `{name}` | missing | | | | | committed exists={committed.exists()}, generated exists={generated.exists()} |"
|
|
)
|
|
continue
|
|
|
|
c_rows, c_cols, c_header = csv_shape(committed)
|
|
g_rows, g_cols, g_header = csv_shape(generated)
|
|
c_hash = normalized_hash(committed)
|
|
g_hash = normalized_hash(generated)
|
|
same = c_hash == g_hash
|
|
status = "match" if same else "diff"
|
|
if not same:
|
|
ok = False
|
|
notes = []
|
|
if c_cols != g_cols:
|
|
notes.append(f"columns {c_cols}!={g_cols}")
|
|
if c_header != g_header:
|
|
notes.append("header differs")
|
|
if c_rows != g_rows:
|
|
notes.append(f"rows {c_rows}!={g_rows}")
|
|
if not same and not notes:
|
|
notes.append(first_difference(committed, generated))
|
|
lines.append(
|
|
f"| `{name}` | {status} | {c_rows} | {g_rows} | `{c_hash[:12]}` | `{g_hash[:12]}` | {'; '.join(notes)} |"
|
|
)
|
|
|
|
lines.append("")
|
|
lines.append("Committed inputs were not modified by this comparison.")
|
|
report_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
|
|
print(f"Wrote comparison report: {report_path}")
|
|
if ok:
|
|
print("Generated inputs match committed inputs.")
|
|
return 0
|
|
print("Generated inputs differ from committed inputs or are missing.")
|
|
return 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|