Remove Python from public release workflow

data-setup/R/01_download_sources.R

@@ -1,5 +0,0 @@
-#!/usr/bin/env Rscript
-
-script <- file.path("data-setup", "download_sources.py")
-status <- system2("python3", script)
-quit(status = status)

data-setup/R/03_compare_generated_inputs.R

@@ -0,0 +1,53 @@
+#!/usr/bin/env Rscript
+
+args <- commandArgs(trailingOnly = FALSE)
+file_arg <- args[grepl("^--file=", args)][1]
+if (!is.na(file_arg)) {
+  script_path <- sub("^--file=", "", file_arg)
+  repo_root <- normalizePath(file.path(dirname(script_path), "..", ".."), mustWork = FALSE)
+} else {
+  repo_root <- normalizePath(getwd(), mustWork = TRUE)
+}
+if (!dir.exists(file.path(repo_root, "data"))) repo_root <- normalizePath(getwd(), mustWork = TRUE)
+
+generated_dir <- Sys.getenv("PARTY2D_GENERATED_INPUT_DIR", file.path(repo_root, "_local", "generated-inputs"))
+report_dir <- Sys.getenv("PARTY2D_REPORT_DIR", file.path(repo_root, "_local", "reports"))
+dir.create(report_dir, recursive = TRUE, showWarnings = FALSE)
+
+files <- c("text_data.csv", "expert.csv", "lr_data.csv", "union_mapping.csv", "party_families.csv")
+
+file_info <- lapply(files, function(file) {
+  committed <- file.path(repo_root, "data", file)
+  generated <- file.path(generated_dir, file)
+  committed_exists <- file.exists(committed)
+  generated_exists <- file.exists(generated)
+  committed_size <- if (committed_exists) file.info(committed)$size else NA_real_
+  generated_size <- if (generated_exists) file.info(generated)$size else NA_real_
+  identical_bytes <- committed_exists && generated_exists && isTRUE(tools::md5sum(committed) == tools::md5sum(generated))
+  data.frame(
+    file = file,
+    committed_exists = committed_exists,
+    generated_exists = generated_exists,
+    committed_size = committed_size,
+    generated_size = generated_size,
+    identical_bytes = identical_bytes,
+    stringsAsFactors = FALSE
+  )
+})
+
+summary <- do.call(rbind, file_info)
+report <- file.path(report_dir, "input_comparison.md")
+lines <- c(
+  "# Model input comparison",
+  "",
+  paste0("Generated: ", format(Sys.time(), "%Y-%m-%d %H:%M:%S %Z")),
+  "",
+  "| File | Committed exists | Generated exists | Committed bytes | Generated bytes | Identical bytes |",
+  "| --- | --- | --- | ---: | ---: | --- |",
+  apply(summary, 1, function(row) {
+    paste0("| ", row[["file"]], " | ", row[["committed_exists"]], " | ", row[["generated_exists"]], " | ", row[["committed_size"]], " | ", row[["generated_size"]], " | ", row[["identical_bytes"]], " |")
+  })
+)
+writeLines(lines, report)
+print(summary, row.names = FALSE)
+message("Comparison report written to ", report)

data-setup/README.md

@@ -1,6 +1,6 @@
 # Data setup
 
-This directory exists because the public repository cannot redistribute the original raw/source files. It downloads the sources that can be fetched automatically, checks user-provided restricted files, rebuilds model-ready inputs, and compares them with the committed files in `../data/`.
+This directory exists because the public repository cannot redistribute the original raw/source files. It checks user-provided source files, rebuilds model-ready inputs, and compares them with the committed files in `../data/`.
 
 The main estimation workflow does not run these scripts. Once the five model-ready CSVs exist in `data/`, fitting and post-estimation are Julia/Stan-only.
 
@@ -8,20 +8,20 @@ The setup workflow never overwrites committed files in `data/`.
 
 Put raw source files in `_local/raw/` or set `PARTY2D_RAW_DATA_DIR` to another local directory. See `source_manifest.csv` and `../docs/RAW_DATA_SOURCES.md` for source-specific access notes.
 
-## What downloads automatically?
+## Source availability
 
-`data-setup/download_sources.py` automatically downloads all source files that are script-accessible under the providers' terms:
+The public repository does not include an automatic source downloader. Users should obtain source files directly from the providers under the providers' terms and place them under `_local/raw/` or another directory selected with `PARTY2D_RAW_DATA_DIR`.
 
-| Source | Automatic? | Requirement |
-| --- | --- | --- |
-| PolDem | Yes | none |
-| PartyFacts crosswalk | Yes | none |
-| CHES family files | Yes | none |
-| POPPA | Yes | none; downloaded from Harvard Dataverse |
-| Global Party Survey 2019 | Yes | none; downloaded from Harvard Dataverse |
-| V-Party | Yes | set `PARTY2D_VDEM_EMAIL`; optionally set `PARTY2D_VDEM_GENDER` |
-| Manifesto Project | Yes, with credentials | set your own `MANIFESTO_API_KEY` or `PARTY2D_MANIFESTO_API_KEY` |
-| Morgan historical expert data | No | place `morgan_positions_raw.csv` locally; available on request |
+| Source | Requirement |
+| --- | --- |
+| PolDem | obtain from provider |
+| PartyFacts crosswalk | obtain from provider |
+| CHES family files | obtain from provider |
+| POPPA | obtain from Harvard Dataverse |
+| Global Party Survey 2019 | obtain from Harvard Dataverse |
+| V-Party | obtain from provider |
+| Manifesto Project | use your own Manifesto Project access/API key |
+| Morgan historical expert data | place `morgan_positions_raw.csv` locally; available on request |
 
 Do not commit downloaded or user-provided source files.
 
@@ -58,10 +58,9 @@ bash data-setup/run_data_setup.sh
 Maintenance options:
 
 - `--dry-run` — check setup scripts only.
-- `--download-only` — download script-accessible sources.
 - `--build-test` — rebuild model-ready inputs from existing local sources.
 - `--compare` — compare regenerated inputs with committed `data/` inputs.
-- no option — download, rebuild, and compare locally.
+- no option — rebuild and compare locally from existing source files.
 
 Minimal check:
 
@@ -69,27 +68,6 @@ Minimal check:
 bash data-setup/run_data_setup.sh --dry-run
 ```
 
-Download sources:
-
-```bash
-bash data-setup/run_data_setup.sh --download-only
-```
-
-V-Party:
-
-```bash
-export PARTY2D_VDEM_EMAIL='you@example.org'
-export PARTY2D_VDEM_GENDER=''   # blank means prefer not to say
-```
-
-Manifesto Project:
-
-```bash
-export MANIFESTO_API_KEY='...'
-# or
-export PARTY2D_MANIFESTO_API_KEY='...'
-```
-
 Morgan is not a public provider download. The local OCR/transcription file can be provided on request and should be placed at:
 
 ```text

data-setup/compare_generated_inputs.py

@@ -1,121 +0,0 @@
-#!/usr/bin/env python3
-"""Compare local regenerated model inputs with committed inputs.
-
-This script is intentionally read-only with respect to data/. It writes a report
-under _local/reports/ and exits nonzero when files differ or are missing.
-"""
-
-from __future__ import annotations
-
-import csv
-import hashlib
-import os
-from pathlib import Path
-from typing import Iterable
-
-
-MODEL_INPUTS = [
-    "text_data.csv",
-    "expert.csv",
-    "lr_data.csv",
-    "union_mapping.csv",
-    "party_families.csv",
-]
-
-
-def normalized_rows(path: Path) -> Iterable[list[str]]:
-    with path.open(newline="", encoding="utf-8") as fh:
-        reader = csv.reader(fh)
-        for row in reader:
-            yield ["" if value is None else value.strip() for value in row]
-
-
-def normalized_hash(path: Path) -> str:
-    h = hashlib.sha256()
-    for row in normalized_rows(path):
-        h.update(("\x1f".join(row) + "\n").encode("utf-8"))
-    return h.hexdigest()
-
-
-def csv_shape(path: Path) -> tuple[int, int, list[str]]:
-    with path.open(newline="", encoding="utf-8") as fh:
-        reader = csv.reader(fh)
-        try:
-            header = next(reader)
-        except StopIteration:
-            return 0, 0, []
-        rows = sum(1 for _ in reader)
-        return rows, len(header), header
-
-
-def first_difference(left: Path, right: Path, max_scan: int = 100000) -> str:
-    for idx, (lrow, rrow) in enumerate(zip(normalized_rows(left), normalized_rows(right)), start=1):
-        if lrow != rrow:
-            return f"line {idx}: committed={lrow[:8]!r} generated={rrow[:8]!r}"
-        if idx >= max_scan:
-            break
-    return "no differing row found in scan window; row counts may differ"
-
-
-def main() -> int:
-    repo_root = Path(__file__).resolve().parents[1]
-    committed_dir = Path(os.environ.get("PARTY2D_COMMITTED_INPUT_DIR", repo_root / "data"))
-    generated_dir = Path(os.environ.get("PARTY2D_GENERATED_INPUT_DIR", repo_root / "_local" / "generated-inputs"))
-    report_dir = Path(os.environ.get("PARTY2D_REPORT_DIR", repo_root / "_local" / "reports"))
-    report_dir.mkdir(parents=True, exist_ok=True)
-    report_path = report_dir / "input_comparison.md"
-
-    lines: list[str] = []
-    lines.append("# Generated input comparison")
-    lines.append("")
-    lines.append(f"Committed input dir: `{committed_dir}`")
-    lines.append(f"Generated input dir: `{generated_dir}`")
-    lines.append("")
-    lines.append("| file | status | committed rows | generated rows | committed hash | generated hash | notes |")
-    lines.append("| --- | --- | ---: | ---: | --- | --- | --- |")
-
-    ok = True
-    for name in MODEL_INPUTS:
-        committed = committed_dir / name
-        generated = generated_dir / name
-        if not committed.exists() or not generated.exists():
-            ok = False
-            lines.append(
-                f"| `{name}` | missing |  |  |  |  | committed exists={committed.exists()}, generated exists={generated.exists()} |"
-            )
-            continue
-
-        c_rows, c_cols, c_header = csv_shape(committed)
-        g_rows, g_cols, g_header = csv_shape(generated)
-        c_hash = normalized_hash(committed)
-        g_hash = normalized_hash(generated)
-        same = c_hash == g_hash
-        status = "match" if same else "diff"
-        if not same:
-            ok = False
-        notes = []
-        if c_cols != g_cols:
-            notes.append(f"columns {c_cols}!={g_cols}")
-        if c_header != g_header:
-            notes.append("header differs")
-        if c_rows != g_rows:
-            notes.append(f"rows {c_rows}!={g_rows}")
-        if not same and not notes:
-            notes.append(first_difference(committed, generated))
-        lines.append(
-            f"| `{name}` | {status} | {c_rows} | {g_rows} | `{c_hash[:12]}` | `{g_hash[:12]}` | {'; '.join(notes)} |"
-        )
-
-    lines.append("")
-    lines.append("Committed inputs were not modified by this comparison.")
-    report_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
-    print(f"Wrote comparison report: {report_path}")
-    if ok:
-        print("Generated inputs match committed inputs.")
-        return 0
-    print("Generated inputs differ from committed inputs or are missing.")
-    return 1
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

data-setup/download_sources.py

@@ -1,240 +0,0 @@
-#!/usr/bin/env python3
-"""Download all script-accessible raw sources into _local/raw/.
-
-The downloader keeps every file inside PARTY2D_RAW_DATA_DIR (default:
-_local/raw). It does not write to committed data/. Sources that require an API
-key or provider form are handled explicitly and reported rather than silently
-skipped.
-"""
-
-from __future__ import annotations
-
-import html.parser
-import json
-import http.cookiejar
-import os
-import re
-import shutil
-import sys
-import urllib.parse
-import urllib.request
-import zipfile
-from pathlib import Path
-
-
-UA = "party2d-data-setup/1.0 (+https://git.seimel.app/armin/party2d)"
-
-
-def raw_dir() -> Path:
-    return Path(os.environ.get("PARTY2D_RAW_DATA_DIR", Path.cwd() / "_local" / "raw")).resolve()
-
-
-def report_dir() -> Path:
-    return Path(os.environ.get("PARTY2D_REPORT_DIR", Path.cwd() / "_local" / "reports")).resolve()
-
-
-def request(url: str, data: bytes | None = None, headers: dict[str, str] | None = None):
-    h = {"User-Agent": UA}
-    if headers:
-        h.update(headers)
-    return urllib.request.Request(url, data=data, headers=h)
-
-
-def download(url: str, dest: Path, *, overwrite: bool = False, headers: dict[str, str] | None = None) -> bool:
-    dest.parent.mkdir(parents=True, exist_ok=True)
-    if dest.exists() and dest.stat().st_size > 0 and not overwrite:
-        print(f"OK existing: {dest}")
-        return True
-    tmp = dest.with_suffix(dest.suffix + ".tmp")
-    print(f"Downloading {url} -> {dest}")
-    try:
-        with urllib.request.urlopen(request(url, headers=headers), timeout=180) as r, tmp.open("wb") as fh:
-            shutil.copyfileobj(r, fh)
-        tmp.replace(dest)
-        return True
-    except Exception as e:
-        if tmp.exists():
-            tmp.unlink()
-        print(f"FAILED {url}: {type(e).__name__}: {e}")
-        return False
-
-
-def dataverse_file_id(doi: str, filename: str, directory: str | None = None) -> int | None:
-    api = "https://dataverse.harvard.edu/api/datasets/:persistentId/?persistentId=" + urllib.parse.quote(doi, safe="")
-    with urllib.request.urlopen(request(api), timeout=120) as r:
-        data = json.load(r)
-    for f in data["data"]["latestVersion"]["files"]:
-        df = f["dataFile"]
-        if df["filename"] == filename and (directory is None or f.get("directoryLabel") == directory):
-            return int(df["id"])
-    return None
-
-
-def download_dataverse(doi: str, filename: str, dest: Path, directory: str | None = None) -> bool:
-    try:
-        fid = dataverse_file_id(doi, filename, directory)
-    except Exception as e:
-        print(f"FAILED Dataverse lookup {doi} {filename}: {type(e).__name__}: {e}")
-        return False
-    if fid is None:
-        print(f"FAILED Dataverse lookup {doi}: file not found: {directory or ''}/{filename}")
-        return False
-    return download(f"https://dataverse.harvard.edu/api/access/datafile/{fid}", dest)
-
-
-class InputParser(html.parser.HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.inputs: dict[str, str] = {}
-
-    def handle_starttag(self, tag, attrs):
-        if tag != "input":
-            return
-        d = dict(attrs)
-        name = d.get("name")
-        if name:
-            self.inputs[name] = d.get("value", "")
-
-
-def download_vparty(dest: Path) -> bool:
-    """Download V-Party R zip through V-Dem's required form.
-
-    V-Dem requires email, gender, privacy acceptance, and format selection. We do
-    not invent personal info: set PARTY2D_VDEM_EMAIL and PARTY2D_VDEM_GENDER
-    (woman/man/non_binary/empty for prefer-not-to-say) to enable this download.
-    """
-    email = os.environ.get("PARTY2D_VDEM_EMAIL")
-    if not email:
-        print("SKIP V-Party: set PARTY2D_VDEM_EMAIL to use V-Dem's required download form")
-        return False
-    gender = os.environ.get("PARTY2D_VDEM_GENDER", "")
-    page = "https://www.v-dem.net/data/v-party-dataset/country-party-date-v2/"
-    cj = http.cookiejar.CookieJar()
-    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
-    try:
-        with opener.open(request(page), timeout=60) as r:
-            html = r.read().decode("utf-8", "replace")
-    except Exception as e:
-        print(f"FAILED V-Party form load: {type(e).__name__}: {e}")
-        return False
-    p = InputParser(); p.feed(html)
-    csrf = p.inputs.get("csrfmiddlewaretoken", "")
-    fields = {
-        "csrfmiddlewaretoken": csrf,
-        "email": email,
-        "gender": gender,
-        "accept_terms": "on",
-        "dataset_file": "17",  # R format in current V-Dem form
-        "website": "",
-    }
-    data = urllib.parse.urlencode(fields).encode()
-    headers = {
-        "Content-Type": "application/x-www-form-urlencoded",
-        "Referer": page,
-        "X-CSRFToken": csrf,
-        "X-Requested-With": "XMLHttpRequest",
-    }
-    dest.parent.mkdir(parents=True, exist_ok=True)
-    tmp = dest.with_suffix(".zip")
-    print(f"Submitting V-Dem form -> {tmp}")
-    try:
-        with opener.open(request(page + "#dataset-download", data=data, headers=headers), timeout=180) as r:
-            payload = r.read()
-            ctype = r.headers.get("content-type", "")
-        # The AJAX form may return JSON with a download URL, or the file itself.
-        if b"download" in payload[:1000] and "json" in ctype:
-            js = json.loads(payload.decode("utf-8"))
-            url = js.get("download_url") or js.get("url") or js.get("file")
-            if url and url.startswith("/"):
-                url = urllib.parse.urljoin(page, url)
-            if not url:
-                print(f"FAILED V-Party form response did not include download URL: {js}")
-                return False
-            if not download(url, tmp, overwrite=True, headers={"Referer": page}):
-                return False
-        else:
-            text = payload.decode("utf-8", "replace")
-            m = re.search(r'href="([^"]*CPD_V-Party_R_v2\.zip[^"]*)"', text)
-            if m:
-                url = urllib.parse.urljoin(page, m.group(1))
-                if not download(url, tmp, overwrite=True, headers={"Referer": page}):
-                    return False
-            else:
-                tmp.write_bytes(payload)
-        if zipfile.is_zipfile(tmp):
-            with zipfile.ZipFile(tmp) as z:
-                r_files = [n for n in z.namelist() if n.lower().endswith((".rds", ".rda", ".rdata"))]
-                if not r_files:
-                    print("FAILED V-Party zip contains no R data file")
-                    return False
-                member = r_files[0]
-                with z.open(member) as src, dest.open("wb") as out:
-                    shutil.copyfileobj(src, out)
-            print(f"Extracted V-Party R data: {dest}")
-            return True
-        print("FAILED V-Party download was not a zip file")
-        return False
-    except Exception as e:
-        print(f"FAILED V-Party form submit: {type(e).__name__}: {e}")
-        return False
-
-
-def download_manifesto(dest: Path) -> bool:
-    key = os.environ.get("MANIFESTO_API_KEY") or os.environ.get("PARTY2D_MANIFESTO_API_KEY")
-    if not key:
-        print("SKIP Manifesto: set MANIFESTO_API_KEY or PARTY2D_MANIFESTO_API_KEY")
-        return False
-    url = "https://manifesto-project.wzb.eu/api/v1/get_core?" + urllib.parse.urlencode({"key": "MPDS2025a", "raw": "true"})
-    return download(
-        url,
-        dest,
-        overwrite=True,
-        headers={"Referer": "https://manifesto-project.wzb.eu/datasets", "API_KEY": key},
-    )
-
-
-def write_report(status: dict[str, bool]) -> None:
-    rd = report_dir(); rd.mkdir(parents=True, exist_ok=True)
-    path = rd / "download_sources_report.md"
-    lines = ["# Download sources report", "", "| source | status |", "| --- | --- |"]
-    for k, v in status.items():
-        lines.append(f"| {k} | {'ok' if v else 'missing/failed'} |")
-    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
-    print(f"Wrote download report: {path}")
-
-
-def main() -> int:
-    root = raw_dir()
-    root.mkdir(parents=True, exist_ok=True)
-    status: dict[str, bool] = {}
-
-    status["PolDem"] = download("https://poldem.eui.eu/downloads/cosa/poldem-election_all.csv", root / "poldem" / "poldem-election_all.csv")
-    status["PartyFacts external parties"] = download("https://partyfacts.herokuapp.com/download/external-parties-csv/", root / "partyfacts" / "partyfacts-external-parties.csv")
-    status["Manifesto MPDS 2025a"] = download_manifesto(root / "manifesto" / "MPDataset_MPDS2025a.csv")
-
-    ches_base = "https://www.chesdata.eu/s/"
-    ches_files = {
-        "1999-2019_CHES_dataset_meansv3.csv": "1999-2019_CHES_dataset_means(v3).csv",
-        "1999-2024_CHES_dataset_meansV2-3k4l.csv": "1999-2024_CHES_dataset_meansV2-3k4l.csv",
-        "CHES_2024_final_v2.csv": "CHES_2024_final_v2.csv",
-        "CHES_2024_ALL_Stacked_Expert.csv": "CHES_2024_expert_level.csv",
-        "CHES_CA2023.csv": "CHES_CA2023.csv",
-        "CHES_CA2023_expert-level.csv": "CHES_CA2023_expert_level.csv",
-        "ches_la_2020_aggregate_level_v01.csv": "ches_la_2020_aggregate_level_v01.csv",
-        "ches_la_2020_expert_level_v01.csv": "CHES_LA2020_expert_level.csv",
-        "CHES_ISRAEL_means_2021_2022.csv": "CHES_ISRAEL_means_2021_2022.csv",
-        "CHES_ISRAEL_expert_level_2021_2022.csv": "CHES_IL_expert_level.csv",
-    }
-    for remote, local in ches_files.items():
-        status[f"CHES {local}"] = download(ches_base + urllib.parse.quote(remote), root / "ches" / local)
-
-    status["POPPA integrated v2"] = download_dataverse("doi:10.7910/DVN/RMQREQ", "poppa_integrated_v2.rds", root / "poppa" / "poppa_integrated_v2.rds", "final_data_v2")
-    status["GPS 2019 party"] = download_dataverse("doi:10.7910/DVN/WMGTNS", "Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab", root / "gps" / "Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab")
-    status["V-Party"] = download_vparty(root / "vparty" / "V-Dem-CPD-Party-V2.rds")
-
-    write_report(status)
-    return 0 if all(status.values()) else 2
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

data-setup/run_data_setup.sh

@@ -7,7 +7,6 @@ Usage: bash data-setup/run_data_setup.sh
 
 Optional maintenance modes:
   --dry-run        check setup scripts only
-  --download-only  download source files where scriptable
   --build-test     rebuild inputs from existing local sources
   --compare        compare regenerated inputs with committed data/
 EOF
@@ -23,7 +22,7 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
 cd "$repo_root"
 
 case "$MODE" in
-  --dry-run|--download-only|--build-test|--compare|--full-test) ;;
+  --dry-run|--build-test|--compare|--full-test) ;;
   *)
     usage
     exit 1
@@ -41,36 +40,20 @@ if [ "$MODE" = "--dry-run" ]; then
   echo "Checking data-setup scripts only; no downloads and no rebuild."
   command -v bash >/dev/null
   command -v Rscript >/dev/null
-  command -v python3 >/dev/null
   bash -n data-setup/check_raw_data.sh
-  python3 -m py_compile data-setup/download_sources.py
-  python3 -m py_compile data-setup/compare_generated_inputs.py
   Rscript -e 'files <- list.files("data-setup/R", pattern="\\.R$", full.names=TRUE); invisible(lapply(files, parse)); cat("R setup scripts parse OK\n")'
   echo "Data setup dry run passed."
   exit 0
 fi
 
-if [ "$MODE" = "--download-only" ]; then
-  python3 data-setup/download_sources.py
-  exit 0
-fi
-
 if [ "$MODE" = "--compare" ]; then
-  python3 data-setup/compare_generated_inputs.py
+  Rscript data-setup/R/03_compare_generated_inputs.R
   exit 0
 fi
 
-if [ "$MODE" = "--full-test" ]; then
-  python3 data-setup/download_sources.py || true
-fi
-
 bash data-setup/check_raw_data.sh
 rm -rf "$PARTY2D_BUILD_DIR" "$PARTY2D_GENERATED_INPUT_DIR"
 mkdir -p "$PARTY2D_BUILD_DIR" "$PARTY2D_GENERATED_INPUT_DIR"
 Rscript data-setup/R/00_install_dependencies.R
 Rscript data-setup/R/02_build_model_inputs.R
-if [ "$MODE" = "--full-test" ]; then
-  python3 data-setup/compare_generated_inputs.py || true
-else
-  python3 data-setup/compare_generated_inputs.py
-fi
+Rscript data-setup/R/03_compare_generated_inputs.R

docs/RAW_DATA_SOURCES.md

@@ -100,15 +100,14 @@ the underlying raw source data.
 
 See `data-setup/source_manifest.csv` for a machine-readable source checklist.
 
-## Scripted download status
+## Source access status
 
-`data-setup/download_sources.py` downloads all sources that are script-accessible
-without project-specific credentials: PolDem, PartyFacts, CHES family files,
-POPPA from Harvard Dataverse, GPS from Harvard Dataverse, and V-Party through the
-provider form when `PARTY2D_VDEM_EMAIL` is set.
+The public repository does not include an automatic source downloader. Obtain
+source files directly from the providers under their terms and place them in the
+documented local raw-data layout before running the rebuild checks.
 
-The Manifesto Project main dataset requires a Manifesto API key/login. Set
-`MANIFESTO_API_KEY` or `PARTY2D_MANIFESTO_API_KEY` for scripted download.
+The Manifesto Project main dataset requires Manifesto Project access/API
+credentials from the provider.
 
 The Morgan historical file is a local OCR/transcription source from Morgan
 (1976), not a public provider download. It can be provided on request and must be

docs/union_mapping.md

@@ -8,7 +8,7 @@ Electoral alliances and blocs are handled in one of two ways:
 1. **Decomposed via mean-constituent averaging** (N=123 mappings): Shared manifesto data feeds into individual constituent party estimates. The output contains the constituents, not the alliance.
 2. **Excluded with documented justification** (see "Excluded Alliance Labels" below): Alliance labels with no mappable constituents are dropped from the output.
 
-A systematic audit of all output parties is provided by `scripts/audit_party_types.py`, which produces `scripts/party_type_audit.csv` classifying every party with evidence. Post-estimation verification in `02_post_estimation.jl` hard-fails if any union/alliance PF ID appears in the output.
+Post-estimation verification in `02_post_estimation.jl` hard-fails if any union/alliance PF ID appears in the output.
 
 ## Overview
 
@@ -339,7 +339,7 @@ These 15 parties have MARPOR entries under both individual (progtype=1/3) and bl
 
 ## Audit Methodology
 
-The audit script `scripts/audit_party_types.py` systematically checks every party in the output CSV:
+The union-mapping audit checks every party in the output CSV:
 
 1. **Union mapping check**: Verifies no `manifesto_pf_id` from `union_mapping.csv` appears in output (hard fail).
 2. **Constituent check**: Identifies parties that are `expert_pf_id` in the mapping (expected: these are individual constituents of unions).
@@ -347,11 +347,4 @@ The audit script `scripts/audit_party_types.py` systematically checks every part
 4. **Name pattern check**: Scans PartyFacts names for alliance indicators (keywords: alliance, coalition, bloc, front, union, alianza, frente; characters: +, /, &).
 5. **Classification**: Each party gets one of: `individual_party`, `flagged_for_review`, `error_union_in_output`.
 
-**To re-run after data updates:**
-```bash
-python3 scripts/audit_party_types.py
-```
-
-Output: `scripts/party_type_audit.csv` with columns: `party_id`, `name`, `country`, `in_union_mapping_as_union`, `in_union_mapping_as_constituent`, `has_expert_data`, `name_flags`, `classification`, `evidence`.
-
 **Post-estimation verification** (`02_post_estimation.jl`): After extracting estimates, loads all `manifesto_pf_id` values from `union_mapping.csv` and checks none appear in the output `party_id` column. If any do, the script errors with a hard fail.

src/sh/show_run_progress.sh

@@ -26,33 +26,7 @@ if [[ -n "$latest_run" ]]; then
   metrics="$latest_run/diagnostics/run_metrics.json"
   if [[ -f "$metrics" ]]; then
     echo "Metrics: $metrics"
-    echo "--- key metrics ---"
-    py="$(command -v python3 || command -v python || true)"
-    if [[ -z "$py" && -x /run/current-system/sw/bin/python3 ]]; then
-      py=/run/current-system/sw/bin/python3
-    fi
-    if [[ -z "$py" ]]; then
-      echo "No Python interpreter found for JSON summary; metrics file is available at: $metrics"
-      exit 0
-    fi
-    "$py" - "$metrics" <<'PY'
-import json, sys
-path = sys.argv[1]
-with open(path) as f:
-    m = json.load(f)
-print("status:", m.get("status"))
-cfg = m.get("config", {})
-print("chains:", cfg.get("num_chains"), "warmup:", cfg.get("num_warmup"), "samples:", cfg.get("num_samples"), "max_depth:", cfg.get("max_depth"), "refresh:", cfg.get("refresh"))
-timing = m.get("timing", {})
-if timing:
-    print("walltime_minutes:", timing.get("walltime_minutes"))
-    print("reported_warmup_seconds:", timing.get("reported_warmup_seconds"))
-    print("reported_sampling_seconds:", timing.get("reported_sampling_seconds"))
-agg = m.get("aggregate", {})
-if agg:
-    print("divergences:", agg.get("divergences"), "treedepth_hits:", agg.get("treedepth_hits"), "mean_leapfrog:", agg.get("mean_leapfrog"))
-print("cmdstan_config_verified:", m.get("cmdstan_config_verified"))
-PY
+    echo "Open the JSON metrics file above for run-summary fields."
   else
     echo "No archived metrics yet. If the run is still sampling, watch the durable log with:"
     if [[ -n "$latest_log" ]]; then