From 9b925f340fa3b650563ae45bd62e6b9832b3eda2 Mon Sep 17 00:00:00 2001 From: Armin Date: Mon, 15 Jun 2026 17:45:03 +0200 Subject: [PATCH] Restore source downloader in R --- README.md | 8 +- data-setup/R/00_install_dependencies.R | 2 +- data-setup/R/01_download_sources.R | 203 +++++++++++++++++++++++++ data-setup/README.md | 50 ++++-- data-setup/run_data_setup.sh | 14 +- docs/RAW_DATA_SOURCES.md | 13 +- 6 files changed, 265 insertions(+), 25 deletions(-) create mode 100644 data-setup/R/01_download_sources.R diff --git a/README.md b/README.md index 8cb26fb..f67dc12 100644 --- a/README.md +++ b/README.md @@ -4,7 +4,7 @@ Code and processed model inputs for generating two-dimensional party-position es ## Repository contents -- `data-setup/` — source-file checks and rebuild workflow for raw files that cannot be redistributed here. +- `data-setup/` — source download, source-file checks, and rebuild workflow for raw files that cannot be redistributed here. - `src/julia/` — Stan data preparation, model fitting, post-estimation, enrichment, and validation. - `models/` — Stan model specification. - `data/` — five processed party-level inputs used by the Julia/Stan model. @@ -20,9 +20,9 @@ Processed inputs needed by the model are included in `data/` so the estimation s The public repository does **not** contain the original raw/source files. Several inputs are third-party datasets with their own terms of use, and the Morgan historical file is a local OCR/transcription source. Instead, this repository provides: 1. committed model-ready inputs in `data/`, sufficient for fitting the Julia/Stan model; and -2. `data-setup/`, a workflow that downloads or checks raw sources and rebuilds comparable model-ready inputs locally. +2. `data-setup/`, an R/Shell workflow that downloads script-accessible sources, checks locally supplied raw sources, and rebuilds comparable model-ready inputs locally. -`data-setup/` documents the required source files and checks locally supplied sources for: +`data-setup/` downloads script-accessible sources and checks locally supplied sources for: - PolDem - PartyFacts crosswalk @@ -44,7 +44,7 @@ Run the full source-data setup workflow with: bash data-setup/run_data_setup.sh ``` -This checks required local files, rebuilds model-ready inputs locally, and writes a comparison report. Manifesto Project requires your own provider access, and the Morgan OCR/transcription file can be provided on request. +This downloads script-accessible source files, checks required local files, rebuilds model-ready inputs locally, and writes a comparison report. Manifesto Project requires your own provider access, and the Morgan OCR/transcription file can be provided on request. ## Running the pipeline diff --git a/data-setup/R/00_install_dependencies.R b/data-setup/R/00_install_dependencies.R index 58b6450..6241b05 100644 --- a/data-setup/R/00_install_dependencies.R +++ b/data-setup/R/00_install_dependencies.R @@ -5,7 +5,7 @@ lib <- Sys.getenv("R_LIBS_USER", file.path(repo_root, "_local", "R", "library")) dir.create(lib, recursive = TRUE, showWarnings = FALSE) .libPaths(c(lib, .libPaths())) -required <- c("tidyverse", "countrycode", "haven", "foreign") +required <- c("tidyverse", "countrycode", "haven", "foreign", "jsonlite") missing <- required[!vapply(required, requireNamespace, quietly = TRUE, FUN.VALUE = logical(1))] if (length(missing) > 0) { diff --git a/data-setup/R/01_download_sources.R b/data-setup/R/01_download_sources.R new file mode 100644 index 0000000..66fd27e --- /dev/null +++ b/data-setup/R/01_download_sources.R @@ -0,0 +1,203 @@ +#!/usr/bin/env Rscript + +args <- commandArgs(trailingOnly = FALSE) +file_arg <- args[grepl("^--file=", args)][1] +if (!is.na(file_arg)) { + script_path <- sub("^--file=", "", file_arg) + repo_root <- normalizePath(file.path(dirname(script_path), "..", ".."), mustWork = FALSE) +} else { + repo_root <- normalizePath(getwd(), mustWork = TRUE) +} +if (!dir.exists(file.path(repo_root, "data-setup"))) repo_root <- normalizePath(getwd(), mustWork = TRUE) + +raw_dir <- normalizePath(Sys.getenv("PARTY2D_RAW_DATA_DIR", file.path(repo_root, "_local", "raw")), mustWork = FALSE) +report_dir <- normalizePath(Sys.getenv("PARTY2D_REPORT_DIR", file.path(repo_root, "_local", "reports")), mustWork = FALSE) +dir.create(raw_dir, recursive = TRUE, showWarnings = FALSE) +dir.create(report_dir, recursive = TRUE, showWarnings = FALSE) + +ua <- "party2d-data-setup/1.0 (+https://git.seimel.app/armin/party2d)" + +download_file <- function(url, dest, overwrite = FALSE, headers = character()) { + dir.create(dirname(dest), recursive = TRUE, showWarnings = FALSE) + if (file.exists(dest) && file.info(dest)$size > 0 && !overwrite) { + message("OK existing: ", dest) + return(TRUE) + } + tmp <- paste0(dest, ".tmp") + if (file.exists(tmp)) unlink(tmp) + message("Downloading ", url, " -> ", dest) + ok <- tryCatch({ + utils::download.file( + url, + tmp, + mode = "wb", + quiet = TRUE, + method = "libcurl", + headers = c("User-Agent" = ua, headers) + ) + file.rename(tmp, dest) + }, error = function(e) { + message("FAILED ", url, ": ", conditionMessage(e)) + FALSE + }) + if (!ok && file.exists(tmp)) unlink(tmp) + isTRUE(ok) +} + +download_dataverse <- function(doi, filename, dest, directory = NA_character_) { + if (!requireNamespace("jsonlite", quietly = TRUE)) { + message("FAILED Dataverse lookup: install R package jsonlite") + return(FALSE) + } + api <- paste0("https://dataverse.harvard.edu/api/datasets/:persistentId/?persistentId=", utils::URLencode(doi, reserved = TRUE)) + fid <- tryCatch({ + meta <- jsonlite::fromJSON(api, simplifyVector = FALSE) + files <- meta$data$latestVersion$files + matches <- Filter(function(x) { + same_file <- identical(x$dataFile$filename, filename) + same_dir <- is.na(directory) || identical(x$directoryLabel, directory) + same_file && same_dir + }, files) + if (length(matches) == 0) NA_integer_ else as.integer(matches[[1]]$dataFile$id) + }, error = function(e) { + message("FAILED Dataverse lookup ", doi, " ", filename, ": ", conditionMessage(e)) + NA_integer_ + }) + if (is.na(fid)) { + message("FAILED Dataverse lookup ", doi, ": file not found: ", filename) + return(FALSE) + } + download_file(paste0("https://dataverse.harvard.edu/api/access/datafile/", fid), dest) +} + +download_manifesto <- function(dest) { + key <- Sys.getenv("MANIFESTO_API_KEY", Sys.getenv("PARTY2D_MANIFESTO_API_KEY", "")) + if (!nzchar(key)) { + message("SKIP Manifesto: set MANIFESTO_API_KEY or PARTY2D_MANIFESTO_API_KEY") + return(FALSE) + } + url <- "https://manifesto-project.wzb.eu/api/v1/get_core?key=MPDS2025a&raw=true" + download_file( + url, + dest, + overwrite = TRUE, + headers = c("Referer" = "https://manifesto-project.wzb.eu/datasets", "API_KEY" = key) + ) +} + +download_vparty <- function(dest) { + email <- Sys.getenv("PARTY2D_VDEM_EMAIL", "") + if (!nzchar(email)) { + message("SKIP V-Party: set PARTY2D_VDEM_EMAIL to use V-Dem's required download form") + return(FALSE) + } + curl <- Sys.which("curl") + if (!nzchar(curl)) { + message("FAILED V-Party: curl is required for the provider form") + return(FALSE) + } + page <- "https://www.v-dem.net/data/v-party-dataset/country-party-date-v2/" + tmpdir <- tempfile("vparty") + dir.create(tmpdir) + on.exit(unlink(tmpdir, recursive = TRUE), add = TRUE) + cookie <- file.path(tmpdir, "cookies.txt") + html <- file.path(tmpdir, "page.html") + payload <- file.path(tmpdir, "payload.bin") + zip_path <- file.path(tmpdir, "vparty.zip") + status <- system2(curl, c("-L", "-A", ua, "-c", cookie, "-b", cookie, "-o", html, page), stdout = TRUE, stderr = TRUE) + if (!file.exists(html)) { + message("FAILED V-Party form load") + return(FALSE) + } + page_text <- paste(readLines(html, warn = FALSE), collapse = "\n") + csrf <- sub('.*name="csrfmiddlewaretoken"[^>]*value="([^"]*)".*', '\\1', page_text) + if (identical(csrf, page_text)) csrf <- "" + gender <- Sys.getenv("PARTY2D_VDEM_GENDER", "") + form <- c( + "csrfmiddlewaretoken", csrf, + "email", email, + "gender", gender, + "accept_terms", "on", + "dataset_file", "17", + "website", "" + ) + args <- c( + "-L", "-A", ua, "-c", cookie, "-b", cookie, + "-H", paste0("Referer: ", page), + "-H", paste0("X-CSRFToken: ", csrf), + "-H", "X-Requested-With: XMLHttpRequest", + "-o", payload, + "--data-urlencode", paste0(form[1], "=", form[2]), + "--data-urlencode", paste0(form[3], "=", form[4]), + "--data-urlencode", paste0(form[5], "=", form[6]), + "--data-urlencode", paste0(form[7], "=", form[8]), + "--data-urlencode", paste0(form[9], "=", form[10]), + "--data-urlencode", paste0(form[11], "=", form[12]), + paste0(page, "#dataset-download") + ) + system2(curl, args, stdout = TRUE, stderr = TRUE) + if (!file.exists(payload) || file.info(payload)$size == 0) { + message("FAILED V-Party form submit") + return(FALSE) + } + bytes <- readBin(payload, "raw", n = min(file.info(payload)$size, 4)) + if (length(bytes) >= 2 && identical(as.integer(bytes[1:2]), c(0x50L, 0x4bL))) { + file.copy(payload, zip_path, overwrite = TRUE) + } else { + text <- paste(readLines(payload, warn = FALSE), collapse = "\n") + m <- regexpr('https?://[^" ]*CPD_V-Party_R_v2\\.zip|/[^" ]*CPD_V-Party_R_v2\\.zip', text) + if (m[1] < 0) { + message("FAILED V-Party form response did not include a recognizable ZIP download") + return(FALSE) + } + url <- regmatches(text, m)[1] + if (startsWith(url, "/")) url <- paste0("https://www.v-dem.net", url) + if (!download_file(url, zip_path, overwrite = TRUE, headers = c("Referer" = page))) return(FALSE) + } + listing <- utils::unzip(zip_path, list = TRUE) + member <- listing$Name[grepl("\\.(rds|rda|rdata)$", listing$Name, ignore.case = TRUE)][1] + if (is.na(member)) { + message("FAILED V-Party ZIP contains no R data file") + return(FALSE) + } + dir.create(dirname(dest), recursive = TRUE, showWarnings = FALSE) + utils::unzip(zip_path, files = member, exdir = tmpdir, overwrite = TRUE) + file.copy(file.path(tmpdir, member), dest, overwrite = TRUE) + message("Extracted V-Party R data: ", dest) + TRUE +} + +status <- c() +status["PolDem"] <- download_file("https://poldem.eui.eu/downloads/cosa/poldem-election_all.csv", file.path(raw_dir, "poldem", "poldem-election_all.csv")) +status["PartyFacts external parties"] <- download_file("https://partyfacts.herokuapp.com/download/external-parties-csv/", file.path(raw_dir, "partyfacts", "partyfacts-external-parties.csv")) +status["Manifesto MPDS 2025a"] <- download_manifesto(file.path(raw_dir, "manifesto", "MPDataset_MPDS2025a.csv")) + +ches_base <- "https://www.chesdata.eu/s/" +ches_files <- c( + "1999-2019_CHES_dataset_meansv3.csv" = "1999-2019_CHES_dataset_means(v3).csv", + "1999-2024_CHES_dataset_meansV2-3k4l.csv" = "1999-2024_CHES_dataset_meansV2-3k4l.csv", + "CHES_2024_final_v2.csv" = "CHES_2024_final_v2.csv", + "CHES_2024_ALL_Stacked_Expert.csv" = "CHES_2024_expert_level.csv", + "CHES_CA2023.csv" = "CHES_CA2023.csv", + "CHES_CA2023_expert-level.csv" = "CHES_CA2023_expert_level.csv", + "ches_la_2020_aggregate_level_v01.csv" = "ches_la_2020_aggregate_level_v01.csv", + "ches_la_2020_expert_level_v01.csv" = "CHES_LA2020_expert_level.csv", + "CHES_ISRAEL_means_2021_2022.csv" = "CHES_ISRAEL_means_2021_2022.csv", + "CHES_ISRAEL_expert_level_2021_2022.csv" = "CHES_IL_expert_level.csv" +) +for (remote in names(ches_files)) { + local <- ches_files[[remote]] + status[paste("CHES", local)] <- download_file(paste0(ches_base, utils::URLencode(remote, reserved = TRUE)), file.path(raw_dir, "ches", local)) +} + +status["POPPA integrated v2"] <- download_dataverse("doi:10.7910/DVN/RMQREQ", "poppa_integrated_v2.rds", file.path(raw_dir, "poppa", "poppa_integrated_v2.rds"), "final_data_v2") +status["GPS 2019 party"] <- download_dataverse("doi:10.7910/DVN/WMGTNS", "Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab", file.path(raw_dir, "gps", "Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab")) +status["V-Party"] <- download_vparty(file.path(raw_dir, "vparty", "V-Dem-CPD-Party-V2.rds")) + +report <- file.path(report_dir, "download_sources_report.md") +lines <- c("# Download sources report", "", "| source | status |", "| --- | --- |") +for (name in names(status)) lines <- c(lines, paste0("| ", name, " | ", if (isTRUE(status[[name]])) "ok" else "missing/failed", " |")) +writeLines(lines, report) +message("Wrote download report: ", report) + +quit(status = if (all(status)) 0 else 2) diff --git a/data-setup/README.md b/data-setup/README.md index d353927..eb447ce 100644 --- a/data-setup/README.md +++ b/data-setup/README.md @@ -1,6 +1,6 @@ # Data setup -This directory exists because the public repository cannot redistribute the original raw/source files. It checks user-provided source files, rebuilds model-ready inputs, and compares them with the committed files in `../data/`. +This directory exists because the public repository cannot redistribute the original raw/source files. It downloads script-accessible sources, checks user-provided source files, rebuilds model-ready inputs, and compares them with the committed files in `../data/`. The main estimation workflow does not run these scripts. Once the five model-ready CSVs exist in `data/`, fitting and post-estimation are Julia/Stan-only. @@ -8,20 +8,20 @@ The setup workflow never overwrites committed files in `data/`. Put raw source files in `_local/raw/` or set `PARTY2D_RAW_DATA_DIR` to another local directory. See `source_manifest.csv` and `../docs/RAW_DATA_SOURCES.md` for source-specific access notes. -## Source availability +## What downloads automatically? -The public repository does not include an automatic source downloader. Users should obtain source files directly from the providers under the providers' terms and place them under `_local/raw/` or another directory selected with `PARTY2D_RAW_DATA_DIR`. +`data-setup/R/01_download_sources.R` downloads source files that are script-accessible under the providers' terms and reports sources that require credentials or manual local files. -| Source | Requirement | -| --- | --- | -| PolDem | obtain from provider | -| PartyFacts crosswalk | obtain from provider | -| CHES family files | obtain from provider | -| POPPA | obtain from Harvard Dataverse | -| Global Party Survey 2019 | obtain from Harvard Dataverse | -| V-Party | obtain from provider | -| Manifesto Project | use your own Manifesto Project access/API key | -| Morgan historical expert data | place `morgan_positions_raw.csv` locally; available on request | +| Source | Automatic? | Requirement | +| --- | --- | --- | +| PolDem | Yes | none | +| PartyFacts crosswalk | Yes | none | +| CHES family files | Yes | none where provider links are live | +| POPPA | Yes | none; downloaded from Harvard Dataverse | +| Global Party Survey 2019 | Yes | none; downloaded from Harvard Dataverse | +| V-Party | Yes, with provider form details | set `PARTY2D_VDEM_EMAIL`; optionally set `PARTY2D_VDEM_GENDER` | +| Manifesto Project | Yes, with credentials | set your own `MANIFESTO_API_KEY` or `PARTY2D_MANIFESTO_API_KEY` | +| Morgan historical expert data | No | place `morgan_positions_raw.csv` locally; available on request | Do not commit downloaded or user-provided source files. @@ -58,9 +58,10 @@ bash data-setup/run_data_setup.sh Maintenance options: - `--dry-run` — check setup scripts only. +- `--download-only` — download script-accessible sources. - `--build-test` — rebuild model-ready inputs from existing local sources. - `--compare` — compare regenerated inputs with committed `data/` inputs. -- no option — rebuild and compare locally from existing source files. +- no option — download, rebuild, and compare locally. Minimal check: @@ -68,6 +69,27 @@ Minimal check: bash data-setup/run_data_setup.sh --dry-run ``` +Download sources: + +```bash +bash data-setup/run_data_setup.sh --download-only +``` + +V-Party: + +```bash +export PARTY2D_VDEM_EMAIL='you@example.org' +export PARTY2D_VDEM_GENDER='' # blank means prefer not to say +``` + +Manifesto Project: + +```bash +export MANIFESTO_API_KEY='...' +# or +export PARTY2D_MANIFESTO_API_KEY='...' +``` + Morgan is not a public provider download. The local OCR/transcription file can be provided on request and should be placed at: ```text diff --git a/data-setup/run_data_setup.sh b/data-setup/run_data_setup.sh index 4db0193..2d38a28 100644 --- a/data-setup/run_data_setup.sh +++ b/data-setup/run_data_setup.sh @@ -7,6 +7,7 @@ Usage: bash data-setup/run_data_setup.sh Optional maintenance modes: --dry-run check setup scripts only + --download-only download source files where scriptable --build-test rebuild inputs from existing local sources --compare compare regenerated inputs with committed data/ EOF @@ -22,7 +23,7 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)" cd "$repo_root" case "$MODE" in - --dry-run|--build-test|--compare|--full-test) ;; + --dry-run|--download-only|--build-test|--compare|--full-test) ;; *) usage exit 1 @@ -51,6 +52,17 @@ if [ "$MODE" = "--compare" ]; then exit 0 fi +if [ "$MODE" = "--download-only" ]; then + Rscript data-setup/R/00_install_dependencies.R + Rscript data-setup/R/01_download_sources.R + exit 0 +fi + +if [ "$MODE" = "--full-test" ]; then + Rscript data-setup/R/00_install_dependencies.R + Rscript data-setup/R/01_download_sources.R || true +fi + bash data-setup/check_raw_data.sh rm -rf "$PARTY2D_BUILD_DIR" "$PARTY2D_GENERATED_INPUT_DIR" mkdir -p "$PARTY2D_BUILD_DIR" "$PARTY2D_GENERATED_INPUT_DIR" diff --git a/docs/RAW_DATA_SOURCES.md b/docs/RAW_DATA_SOURCES.md index aa4e6cc..57c5cd4 100644 --- a/docs/RAW_DATA_SOURCES.md +++ b/docs/RAW_DATA_SOURCES.md @@ -100,14 +100,17 @@ the underlying raw source data. See `data-setup/source_manifest.csv` for a machine-readable source checklist. -## Source access status +## Scripted download status -The public repository does not include an automatic source downloader. Obtain -source files directly from the providers under their terms and place them in the -documented local raw-data layout before running the rebuild checks. +`data-setup/R/01_download_sources.R` downloads sources that are script-accessible +under provider terms: PolDem, PartyFacts, CHES family files where provider links +are live, POPPA from Harvard Dataverse, GPS from Harvard Dataverse, Manifesto +with user credentials, and V-Party through the provider form when +`PARTY2D_VDEM_EMAIL` is set. The Manifesto Project main dataset requires Manifesto Project access/API -credentials from the provider. +credentials from the provider. Set `MANIFESTO_API_KEY` or +`PARTY2D_MANIFESTO_API_KEY` for scripted download. The Morgan historical file is a local OCR/transcription source from Morgan (1976), not a public provider download. It can be provided on request and must be