From 952329bff40a5a2368e2a42eb808b050fc6ed32f Mon Sep 17 00:00:00 2001 From: Armin Date: Mon, 15 Jun 2026 17:38:59 +0200 Subject: [PATCH] Remove Python from public release workflow --- data-setup/R/01_download_sources.R | 5 - data-setup/R/03_compare_generated_inputs.R | 53 +++++ data-setup/README.md | 50 ++--- data-setup/compare_generated_inputs.py | 121 ----------- data-setup/download_sources.py | 240 --------------------- data-setup/run_data_setup.sh | 23 +- docs/RAW_DATA_SOURCES.md | 13 +- docs/union_mapping.md | 11 +- src/sh/show_run_progress.sh | 28 +-- 9 files changed, 79 insertions(+), 465 deletions(-) delete mode 100644 data-setup/R/01_download_sources.R create mode 100644 data-setup/R/03_compare_generated_inputs.R delete mode 100644 data-setup/compare_generated_inputs.py delete mode 100644 data-setup/download_sources.py diff --git a/data-setup/R/01_download_sources.R b/data-setup/R/01_download_sources.R deleted file mode 100644 index a6c12bd..0000000 --- a/data-setup/R/01_download_sources.R +++ /dev/null @@ -1,5 +0,0 @@ -#!/usr/bin/env Rscript - -script <- file.path("data-setup", "download_sources.py") -status <- system2("python3", script) -quit(status = status) diff --git a/data-setup/R/03_compare_generated_inputs.R b/data-setup/R/03_compare_generated_inputs.R new file mode 100644 index 0000000..584c352 --- /dev/null +++ b/data-setup/R/03_compare_generated_inputs.R @@ -0,0 +1,53 @@ +#!/usr/bin/env Rscript + +args <- commandArgs(trailingOnly = FALSE) +file_arg <- args[grepl("^--file=", args)][1] +if (!is.na(file_arg)) { + script_path <- sub("^--file=", "", file_arg) + repo_root <- normalizePath(file.path(dirname(script_path), "..", ".."), mustWork = FALSE) +} else { + repo_root <- normalizePath(getwd(), mustWork = TRUE) +} +if (!dir.exists(file.path(repo_root, "data"))) repo_root <- normalizePath(getwd(), mustWork = TRUE) + +generated_dir <- Sys.getenv("PARTY2D_GENERATED_INPUT_DIR", file.path(repo_root, "_local", "generated-inputs")) +report_dir <- Sys.getenv("PARTY2D_REPORT_DIR", file.path(repo_root, "_local", "reports")) +dir.create(report_dir, recursive = TRUE, showWarnings = FALSE) + +files <- c("text_data.csv", "expert.csv", "lr_data.csv", "union_mapping.csv", "party_families.csv") + +file_info <- lapply(files, function(file) { + committed <- file.path(repo_root, "data", file) + generated <- file.path(generated_dir, file) + committed_exists <- file.exists(committed) + generated_exists <- file.exists(generated) + committed_size <- if (committed_exists) file.info(committed)$size else NA_real_ + generated_size <- if (generated_exists) file.info(generated)$size else NA_real_ + identical_bytes <- committed_exists && generated_exists && isTRUE(tools::md5sum(committed) == tools::md5sum(generated)) + data.frame( + file = file, + committed_exists = committed_exists, + generated_exists = generated_exists, + committed_size = committed_size, + generated_size = generated_size, + identical_bytes = identical_bytes, + stringsAsFactors = FALSE + ) +}) + +summary <- do.call(rbind, file_info) +report <- file.path(report_dir, "input_comparison.md") +lines <- c( + "# Model input comparison", + "", + paste0("Generated: ", format(Sys.time(), "%Y-%m-%d %H:%M:%S %Z")), + "", + "| File | Committed exists | Generated exists | Committed bytes | Generated bytes | Identical bytes |", + "| --- | --- | --- | ---: | ---: | --- |", + apply(summary, 1, function(row) { + paste0("| ", row[["file"]], " | ", row[["committed_exists"]], " | ", row[["generated_exists"]], " | ", row[["committed_size"]], " | ", row[["generated_size"]], " | ", row[["identical_bytes"]], " |") + }) +) +writeLines(lines, report) +print(summary, row.names = FALSE) +message("Comparison report written to ", report) diff --git a/data-setup/README.md b/data-setup/README.md index 7a7d997..d353927 100644 --- a/data-setup/README.md +++ b/data-setup/README.md @@ -1,6 +1,6 @@ # Data setup -This directory exists because the public repository cannot redistribute the original raw/source files. It downloads the sources that can be fetched automatically, checks user-provided restricted files, rebuilds model-ready inputs, and compares them with the committed files in `../data/`. +This directory exists because the public repository cannot redistribute the original raw/source files. It checks user-provided source files, rebuilds model-ready inputs, and compares them with the committed files in `../data/`. The main estimation workflow does not run these scripts. Once the five model-ready CSVs exist in `data/`, fitting and post-estimation are Julia/Stan-only. @@ -8,20 +8,20 @@ The setup workflow never overwrites committed files in `data/`. Put raw source files in `_local/raw/` or set `PARTY2D_RAW_DATA_DIR` to another local directory. See `source_manifest.csv` and `../docs/RAW_DATA_SOURCES.md` for source-specific access notes. -## What downloads automatically? +## Source availability -`data-setup/download_sources.py` automatically downloads all source files that are script-accessible under the providers' terms: +The public repository does not include an automatic source downloader. Users should obtain source files directly from the providers under the providers' terms and place them under `_local/raw/` or another directory selected with `PARTY2D_RAW_DATA_DIR`. -| Source | Automatic? | Requirement | -| --- | --- | --- | -| PolDem | Yes | none | -| PartyFacts crosswalk | Yes | none | -| CHES family files | Yes | none | -| POPPA | Yes | none; downloaded from Harvard Dataverse | -| Global Party Survey 2019 | Yes | none; downloaded from Harvard Dataverse | -| V-Party | Yes | set `PARTY2D_VDEM_EMAIL`; optionally set `PARTY2D_VDEM_GENDER` | -| Manifesto Project | Yes, with credentials | set your own `MANIFESTO_API_KEY` or `PARTY2D_MANIFESTO_API_KEY` | -| Morgan historical expert data | No | place `morgan_positions_raw.csv` locally; available on request | +| Source | Requirement | +| --- | --- | +| PolDem | obtain from provider | +| PartyFacts crosswalk | obtain from provider | +| CHES family files | obtain from provider | +| POPPA | obtain from Harvard Dataverse | +| Global Party Survey 2019 | obtain from Harvard Dataverse | +| V-Party | obtain from provider | +| Manifesto Project | use your own Manifesto Project access/API key | +| Morgan historical expert data | place `morgan_positions_raw.csv` locally; available on request | Do not commit downloaded or user-provided source files. @@ -58,10 +58,9 @@ bash data-setup/run_data_setup.sh Maintenance options: - `--dry-run` — check setup scripts only. -- `--download-only` — download script-accessible sources. - `--build-test` — rebuild model-ready inputs from existing local sources. - `--compare` — compare regenerated inputs with committed `data/` inputs. -- no option — download, rebuild, and compare locally. +- no option — rebuild and compare locally from existing source files. Minimal check: @@ -69,27 +68,6 @@ Minimal check: bash data-setup/run_data_setup.sh --dry-run ``` -Download sources: - -```bash -bash data-setup/run_data_setup.sh --download-only -``` - -V-Party: - -```bash -export PARTY2D_VDEM_EMAIL='you@example.org' -export PARTY2D_VDEM_GENDER='' # blank means prefer not to say -``` - -Manifesto Project: - -```bash -export MANIFESTO_API_KEY='...' -# or -export PARTY2D_MANIFESTO_API_KEY='...' -``` - Morgan is not a public provider download. The local OCR/transcription file can be provided on request and should be placed at: ```text diff --git a/data-setup/compare_generated_inputs.py b/data-setup/compare_generated_inputs.py deleted file mode 100644 index 94a05c0..0000000 --- a/data-setup/compare_generated_inputs.py +++ /dev/null @@ -1,121 +0,0 @@ -#!/usr/bin/env python3 -"""Compare local regenerated model inputs with committed inputs. - -This script is intentionally read-only with respect to data/. It writes a report -under _local/reports/ and exits nonzero when files differ or are missing. -""" - -from __future__ import annotations - -import csv -import hashlib -import os -from pathlib import Path -from typing import Iterable - - -MODEL_INPUTS = [ - "text_data.csv", - "expert.csv", - "lr_data.csv", - "union_mapping.csv", - "party_families.csv", -] - - -def normalized_rows(path: Path) -> Iterable[list[str]]: - with path.open(newline="", encoding="utf-8") as fh: - reader = csv.reader(fh) - for row in reader: - yield ["" if value is None else value.strip() for value in row] - - -def normalized_hash(path: Path) -> str: - h = hashlib.sha256() - for row in normalized_rows(path): - h.update(("\x1f".join(row) + "\n").encode("utf-8")) - return h.hexdigest() - - -def csv_shape(path: Path) -> tuple[int, int, list[str]]: - with path.open(newline="", encoding="utf-8") as fh: - reader = csv.reader(fh) - try: - header = next(reader) - except StopIteration: - return 0, 0, [] - rows = sum(1 for _ in reader) - return rows, len(header), header - - -def first_difference(left: Path, right: Path, max_scan: int = 100000) -> str: - for idx, (lrow, rrow) in enumerate(zip(normalized_rows(left), normalized_rows(right)), start=1): - if lrow != rrow: - return f"line {idx}: committed={lrow[:8]!r} generated={rrow[:8]!r}" - if idx >= max_scan: - break - return "no differing row found in scan window; row counts may differ" - - -def main() -> int: - repo_root = Path(__file__).resolve().parents[1] - committed_dir = Path(os.environ.get("PARTY2D_COMMITTED_INPUT_DIR", repo_root / "data")) - generated_dir = Path(os.environ.get("PARTY2D_GENERATED_INPUT_DIR", repo_root / "_local" / "generated-inputs")) - report_dir = Path(os.environ.get("PARTY2D_REPORT_DIR", repo_root / "_local" / "reports")) - report_dir.mkdir(parents=True, exist_ok=True) - report_path = report_dir / "input_comparison.md" - - lines: list[str] = [] - lines.append("# Generated input comparison") - lines.append("") - lines.append(f"Committed input dir: `{committed_dir}`") - lines.append(f"Generated input dir: `{generated_dir}`") - lines.append("") - lines.append("| file | status | committed rows | generated rows | committed hash | generated hash | notes |") - lines.append("| --- | --- | ---: | ---: | --- | --- | --- |") - - ok = True - for name in MODEL_INPUTS: - committed = committed_dir / name - generated = generated_dir / name - if not committed.exists() or not generated.exists(): - ok = False - lines.append( - f"| `{name}` | missing | | | | | committed exists={committed.exists()}, generated exists={generated.exists()} |" - ) - continue - - c_rows, c_cols, c_header = csv_shape(committed) - g_rows, g_cols, g_header = csv_shape(generated) - c_hash = normalized_hash(committed) - g_hash = normalized_hash(generated) - same = c_hash == g_hash - status = "match" if same else "diff" - if not same: - ok = False - notes = [] - if c_cols != g_cols: - notes.append(f"columns {c_cols}!={g_cols}") - if c_header != g_header: - notes.append("header differs") - if c_rows != g_rows: - notes.append(f"rows {c_rows}!={g_rows}") - if not same and not notes: - notes.append(first_difference(committed, generated)) - lines.append( - f"| `{name}` | {status} | {c_rows} | {g_rows} | `{c_hash[:12]}` | `{g_hash[:12]}` | {'; '.join(notes)} |" - ) - - lines.append("") - lines.append("Committed inputs were not modified by this comparison.") - report_path.write_text("\n".join(lines) + "\n", encoding="utf-8") - print(f"Wrote comparison report: {report_path}") - if ok: - print("Generated inputs match committed inputs.") - return 0 - print("Generated inputs differ from committed inputs or are missing.") - return 1 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/data-setup/download_sources.py b/data-setup/download_sources.py deleted file mode 100644 index 40a7642..0000000 --- a/data-setup/download_sources.py +++ /dev/null @@ -1,240 +0,0 @@ -#!/usr/bin/env python3 -"""Download all script-accessible raw sources into _local/raw/. - -The downloader keeps every file inside PARTY2D_RAW_DATA_DIR (default: -_local/raw). It does not write to committed data/. Sources that require an API -key or provider form are handled explicitly and reported rather than silently -skipped. -""" - -from __future__ import annotations - -import html.parser -import json -import http.cookiejar -import os -import re -import shutil -import sys -import urllib.parse -import urllib.request -import zipfile -from pathlib import Path - - -UA = "party2d-data-setup/1.0 (+https://git.seimel.app/armin/party2d)" - - -def raw_dir() -> Path: - return Path(os.environ.get("PARTY2D_RAW_DATA_DIR", Path.cwd() / "_local" / "raw")).resolve() - - -def report_dir() -> Path: - return Path(os.environ.get("PARTY2D_REPORT_DIR", Path.cwd() / "_local" / "reports")).resolve() - - -def request(url: str, data: bytes | None = None, headers: dict[str, str] | None = None): - h = {"User-Agent": UA} - if headers: - h.update(headers) - return urllib.request.Request(url, data=data, headers=h) - - -def download(url: str, dest: Path, *, overwrite: bool = False, headers: dict[str, str] | None = None) -> bool: - dest.parent.mkdir(parents=True, exist_ok=True) - if dest.exists() and dest.stat().st_size > 0 and not overwrite: - print(f"OK existing: {dest}") - return True - tmp = dest.with_suffix(dest.suffix + ".tmp") - print(f"Downloading {url} -> {dest}") - try: - with urllib.request.urlopen(request(url, headers=headers), timeout=180) as r, tmp.open("wb") as fh: - shutil.copyfileobj(r, fh) - tmp.replace(dest) - return True - except Exception as e: - if tmp.exists(): - tmp.unlink() - print(f"FAILED {url}: {type(e).__name__}: {e}") - return False - - -def dataverse_file_id(doi: str, filename: str, directory: str | None = None) -> int | None: - api = "https://dataverse.harvard.edu/api/datasets/:persistentId/?persistentId=" + urllib.parse.quote(doi, safe="") - with urllib.request.urlopen(request(api), timeout=120) as r: - data = json.load(r) - for f in data["data"]["latestVersion"]["files"]: - df = f["dataFile"] - if df["filename"] == filename and (directory is None or f.get("directoryLabel") == directory): - return int(df["id"]) - return None - - -def download_dataverse(doi: str, filename: str, dest: Path, directory: str | None = None) -> bool: - try: - fid = dataverse_file_id(doi, filename, directory) - except Exception as e: - print(f"FAILED Dataverse lookup {doi} {filename}: {type(e).__name__}: {e}") - return False - if fid is None: - print(f"FAILED Dataverse lookup {doi}: file not found: {directory or ''}/{filename}") - return False - return download(f"https://dataverse.harvard.edu/api/access/datafile/{fid}", dest) - - -class InputParser(html.parser.HTMLParser): - def __init__(self): - super().__init__() - self.inputs: dict[str, str] = {} - - def handle_starttag(self, tag, attrs): - if tag != "input": - return - d = dict(attrs) - name = d.get("name") - if name: - self.inputs[name] = d.get("value", "") - - -def download_vparty(dest: Path) -> bool: - """Download V-Party R zip through V-Dem's required form. - - V-Dem requires email, gender, privacy acceptance, and format selection. We do - not invent personal info: set PARTY2D_VDEM_EMAIL and PARTY2D_VDEM_GENDER - (woman/man/non_binary/empty for prefer-not-to-say) to enable this download. - """ - email = os.environ.get("PARTY2D_VDEM_EMAIL") - if not email: - print("SKIP V-Party: set PARTY2D_VDEM_EMAIL to use V-Dem's required download form") - return False - gender = os.environ.get("PARTY2D_VDEM_GENDER", "") - page = "https://www.v-dem.net/data/v-party-dataset/country-party-date-v2/" - cj = http.cookiejar.CookieJar() - opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj)) - try: - with opener.open(request(page), timeout=60) as r: - html = r.read().decode("utf-8", "replace") - except Exception as e: - print(f"FAILED V-Party form load: {type(e).__name__}: {e}") - return False - p = InputParser(); p.feed(html) - csrf = p.inputs.get("csrfmiddlewaretoken", "") - fields = { - "csrfmiddlewaretoken": csrf, - "email": email, - "gender": gender, - "accept_terms": "on", - "dataset_file": "17", # R format in current V-Dem form - "website": "", - } - data = urllib.parse.urlencode(fields).encode() - headers = { - "Content-Type": "application/x-www-form-urlencoded", - "Referer": page, - "X-CSRFToken": csrf, - "X-Requested-With": "XMLHttpRequest", - } - dest.parent.mkdir(parents=True, exist_ok=True) - tmp = dest.with_suffix(".zip") - print(f"Submitting V-Dem form -> {tmp}") - try: - with opener.open(request(page + "#dataset-download", data=data, headers=headers), timeout=180) as r: - payload = r.read() - ctype = r.headers.get("content-type", "") - # The AJAX form may return JSON with a download URL, or the file itself. - if b"download" in payload[:1000] and "json" in ctype: - js = json.loads(payload.decode("utf-8")) - url = js.get("download_url") or js.get("url") or js.get("file") - if url and url.startswith("/"): - url = urllib.parse.urljoin(page, url) - if not url: - print(f"FAILED V-Party form response did not include download URL: {js}") - return False - if not download(url, tmp, overwrite=True, headers={"Referer": page}): - return False - else: - text = payload.decode("utf-8", "replace") - m = re.search(r'href="([^"]*CPD_V-Party_R_v2\.zip[^"]*)"', text) - if m: - url = urllib.parse.urljoin(page, m.group(1)) - if not download(url, tmp, overwrite=True, headers={"Referer": page}): - return False - else: - tmp.write_bytes(payload) - if zipfile.is_zipfile(tmp): - with zipfile.ZipFile(tmp) as z: - r_files = [n for n in z.namelist() if n.lower().endswith((".rds", ".rda", ".rdata"))] - if not r_files: - print("FAILED V-Party zip contains no R data file") - return False - member = r_files[0] - with z.open(member) as src, dest.open("wb") as out: - shutil.copyfileobj(src, out) - print(f"Extracted V-Party R data: {dest}") - return True - print("FAILED V-Party download was not a zip file") - return False - except Exception as e: - print(f"FAILED V-Party form submit: {type(e).__name__}: {e}") - return False - - -def download_manifesto(dest: Path) -> bool: - key = os.environ.get("MANIFESTO_API_KEY") or os.environ.get("PARTY2D_MANIFESTO_API_KEY") - if not key: - print("SKIP Manifesto: set MANIFESTO_API_KEY or PARTY2D_MANIFESTO_API_KEY") - return False - url = "https://manifesto-project.wzb.eu/api/v1/get_core?" + urllib.parse.urlencode({"key": "MPDS2025a", "raw": "true"}) - return download( - url, - dest, - overwrite=True, - headers={"Referer": "https://manifesto-project.wzb.eu/datasets", "API_KEY": key}, - ) - - -def write_report(status: dict[str, bool]) -> None: - rd = report_dir(); rd.mkdir(parents=True, exist_ok=True) - path = rd / "download_sources_report.md" - lines = ["# Download sources report", "", "| source | status |", "| --- | --- |"] - for k, v in status.items(): - lines.append(f"| {k} | {'ok' if v else 'missing/failed'} |") - path.write_text("\n".join(lines) + "\n", encoding="utf-8") - print(f"Wrote download report: {path}") - - -def main() -> int: - root = raw_dir() - root.mkdir(parents=True, exist_ok=True) - status: dict[str, bool] = {} - - status["PolDem"] = download("https://poldem.eui.eu/downloads/cosa/poldem-election_all.csv", root / "poldem" / "poldem-election_all.csv") - status["PartyFacts external parties"] = download("https://partyfacts.herokuapp.com/download/external-parties-csv/", root / "partyfacts" / "partyfacts-external-parties.csv") - status["Manifesto MPDS 2025a"] = download_manifesto(root / "manifesto" / "MPDataset_MPDS2025a.csv") - - ches_base = "https://www.chesdata.eu/s/" - ches_files = { - "1999-2019_CHES_dataset_meansv3.csv": "1999-2019_CHES_dataset_means(v3).csv", - "1999-2024_CHES_dataset_meansV2-3k4l.csv": "1999-2024_CHES_dataset_meansV2-3k4l.csv", - "CHES_2024_final_v2.csv": "CHES_2024_final_v2.csv", - "CHES_2024_ALL_Stacked_Expert.csv": "CHES_2024_expert_level.csv", - "CHES_CA2023.csv": "CHES_CA2023.csv", - "CHES_CA2023_expert-level.csv": "CHES_CA2023_expert_level.csv", - "ches_la_2020_aggregate_level_v01.csv": "ches_la_2020_aggregate_level_v01.csv", - "ches_la_2020_expert_level_v01.csv": "CHES_LA2020_expert_level.csv", - "CHES_ISRAEL_means_2021_2022.csv": "CHES_ISRAEL_means_2021_2022.csv", - "CHES_ISRAEL_expert_level_2021_2022.csv": "CHES_IL_expert_level.csv", - } - for remote, local in ches_files.items(): - status[f"CHES {local}"] = download(ches_base + urllib.parse.quote(remote), root / "ches" / local) - - status["POPPA integrated v2"] = download_dataverse("doi:10.7910/DVN/RMQREQ", "poppa_integrated_v2.rds", root / "poppa" / "poppa_integrated_v2.rds", "final_data_v2") - status["GPS 2019 party"] = download_dataverse("doi:10.7910/DVN/WMGTNS", "Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab", root / "gps" / "Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab") - status["V-Party"] = download_vparty(root / "vparty" / "V-Dem-CPD-Party-V2.rds") - - write_report(status) - return 0 if all(status.values()) else 2 - - -if __name__ == "__main__": - raise SystemExit(main()) diff --git a/data-setup/run_data_setup.sh b/data-setup/run_data_setup.sh index 655d6f0..4db0193 100644 --- a/data-setup/run_data_setup.sh +++ b/data-setup/run_data_setup.sh @@ -7,7 +7,6 @@ Usage: bash data-setup/run_data_setup.sh Optional maintenance modes: --dry-run check setup scripts only - --download-only download source files where scriptable --build-test rebuild inputs from existing local sources --compare compare regenerated inputs with committed data/ EOF @@ -23,7 +22,7 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" cd "$repo_root" case "$MODE" in - --dry-run|--download-only|--build-test|--compare|--full-test) ;; + --dry-run|--build-test|--compare|--full-test) ;; *) usage exit 1 @@ -41,36 +40,20 @@ if [ "$MODE" = "--dry-run" ]; then echo "Checking data-setup scripts only; no downloads and no rebuild." command -v bash >/dev/null command -v Rscript >/dev/null - command -v python3 >/dev/null bash -n data-setup/check_raw_data.sh - python3 -m py_compile data-setup/download_sources.py - python3 -m py_compile data-setup/compare_generated_inputs.py Rscript -e 'files <- list.files("data-setup/R", pattern="\\.R$", full.names=TRUE); invisible(lapply(files, parse)); cat("R setup scripts parse OK\n")' echo "Data setup dry run passed." exit 0 fi -if [ "$MODE" = "--download-only" ]; then - python3 data-setup/download_sources.py - exit 0 -fi - if [ "$MODE" = "--compare" ]; then - python3 data-setup/compare_generated_inputs.py + Rscript data-setup/R/03_compare_generated_inputs.R exit 0 fi -if [ "$MODE" = "--full-test" ]; then - python3 data-setup/download_sources.py || true -fi - bash data-setup/check_raw_data.sh rm -rf "$PARTY2D_BUILD_DIR" "$PARTY2D_GENERATED_INPUT_DIR" mkdir -p "$PARTY2D_BUILD_DIR" "$PARTY2D_GENERATED_INPUT_DIR" Rscript data-setup/R/00_install_dependencies.R Rscript data-setup/R/02_build_model_inputs.R -if [ "$MODE" = "--full-test" ]; then - python3 data-setup/compare_generated_inputs.py || true -else - python3 data-setup/compare_generated_inputs.py -fi +Rscript data-setup/R/03_compare_generated_inputs.R diff --git a/docs/RAW_DATA_SOURCES.md b/docs/RAW_DATA_SOURCES.md index fa968bd..aa4e6cc 100644 --- a/docs/RAW_DATA_SOURCES.md +++ b/docs/RAW_DATA_SOURCES.md @@ -100,15 +100,14 @@ the underlying raw source data. See `data-setup/source_manifest.csv` for a machine-readable source checklist. -## Scripted download status +## Source access status -`data-setup/download_sources.py` downloads all sources that are script-accessible -without project-specific credentials: PolDem, PartyFacts, CHES family files, -POPPA from Harvard Dataverse, GPS from Harvard Dataverse, and V-Party through the -provider form when `PARTY2D_VDEM_EMAIL` is set. +The public repository does not include an automatic source downloader. Obtain +source files directly from the providers under their terms and place them in the +documented local raw-data layout before running the rebuild checks. -The Manifesto Project main dataset requires a Manifesto API key/login. Set -`MANIFESTO_API_KEY` or `PARTY2D_MANIFESTO_API_KEY` for scripted download. +The Manifesto Project main dataset requires Manifesto Project access/API +credentials from the provider. The Morgan historical file is a local OCR/transcription source from Morgan (1976), not a public provider download. It can be provided on request and must be diff --git a/docs/union_mapping.md b/docs/union_mapping.md index 1f35638..43e078d 100644 --- a/docs/union_mapping.md +++ b/docs/union_mapping.md @@ -8,7 +8,7 @@ Electoral alliances and blocs are handled in one of two ways: 1. **Decomposed via mean-constituent averaging** (N=123 mappings): Shared manifesto data feeds into individual constituent party estimates. The output contains the constituents, not the alliance. 2. **Excluded with documented justification** (see "Excluded Alliance Labels" below): Alliance labels with no mappable constituents are dropped from the output. -A systematic audit of all output parties is provided by `scripts/audit_party_types.py`, which produces `scripts/party_type_audit.csv` classifying every party with evidence. Post-estimation verification in `02_post_estimation.jl` hard-fails if any union/alliance PF ID appears in the output. +Post-estimation verification in `02_post_estimation.jl` hard-fails if any union/alliance PF ID appears in the output. ## Overview @@ -339,7 +339,7 @@ These 15 parties have MARPOR entries under both individual (progtype=1/3) and bl ## Audit Methodology -The audit script `scripts/audit_party_types.py` systematically checks every party in the output CSV: +The union-mapping audit checks every party in the output CSV: 1. **Union mapping check**: Verifies no `manifesto_pf_id` from `union_mapping.csv` appears in output (hard fail). 2. **Constituent check**: Identifies parties that are `expert_pf_id` in the mapping (expected: these are individual constituents of unions). @@ -347,11 +347,4 @@ The audit script `scripts/audit_party_types.py` systematically checks every part 4. **Name pattern check**: Scans PartyFacts names for alliance indicators (keywords: alliance, coalition, bloc, front, union, alianza, frente; characters: +, /, &). 5. **Classification**: Each party gets one of: `individual_party`, `flagged_for_review`, `error_union_in_output`. -**To re-run after data updates:** -```bash -python3 scripts/audit_party_types.py -``` - -Output: `scripts/party_type_audit.csv` with columns: `party_id`, `name`, `country`, `in_union_mapping_as_union`, `in_union_mapping_as_constituent`, `has_expert_data`, `name_flags`, `classification`, `evidence`. - **Post-estimation verification** (`02_post_estimation.jl`): After extracting estimates, loads all `manifesto_pf_id` values from `union_mapping.csv` and checks none appear in the output `party_id` column. If any do, the script errors with a hard fail. diff --git a/src/sh/show_run_progress.sh b/src/sh/show_run_progress.sh index 4178c19..3df2b6a 100755 --- a/src/sh/show_run_progress.sh +++ b/src/sh/show_run_progress.sh @@ -26,33 +26,7 @@ if [[ -n "$latest_run" ]]; then metrics="$latest_run/diagnostics/run_metrics.json" if [[ -f "$metrics" ]]; then echo "Metrics: $metrics" - echo "--- key metrics ---" - py="$(command -v python3 || command -v python || true)" - if [[ -z "$py" && -x /run/current-system/sw/bin/python3 ]]; then - py=/run/current-system/sw/bin/python3 - fi - if [[ -z "$py" ]]; then - echo "No Python interpreter found for JSON summary; metrics file is available at: $metrics" - exit 0 - fi - "$py" - "$metrics" <<'PY' -import json, sys -path = sys.argv[1] -with open(path) as f: - m = json.load(f) -print("status:", m.get("status")) -cfg = m.get("config", {}) -print("chains:", cfg.get("num_chains"), "warmup:", cfg.get("num_warmup"), "samples:", cfg.get("num_samples"), "max_depth:", cfg.get("max_depth"), "refresh:", cfg.get("refresh")) -timing = m.get("timing", {}) -if timing: - print("walltime_minutes:", timing.get("walltime_minutes")) - print("reported_warmup_seconds:", timing.get("reported_warmup_seconds")) - print("reported_sampling_seconds:", timing.get("reported_sampling_seconds")) -agg = m.get("aggregate", {}) -if agg: - print("divergences:", agg.get("divergences"), "treedepth_hits:", agg.get("treedepth_hits"), "mean_leapfrog:", agg.get("mean_leapfrog")) -print("cmdstan_config_verified:", m.get("cmdstan_config_verified")) -PY + echo "Open the JSON metrics file above for run-summary fields." else echo "No archived metrics yet. If the run is still sampling, watch the durable log with:" if [[ -n "$latest_log" ]]; then