armin/party2d
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

Restore source downloader in R

Browse Source
This commit is contained in:
Armin
2026-06-15 17:45:03 +02:00
parent dad75063fc
commit 9b925f340f
6 changed files with 265 additions and 25 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+4 -4
View File
@@ -4,7 +4,7 @@ Code and processed model inputs for generating two-dimensional party-position es
## Repository contents
- `data-setup/` — source-file checks and rebuild workflow for raw files that cannot be redistributed here.
- `data-setup/` — source download, source-file checks, and rebuild workflow for raw files that cannot be redistributed here.
- `src/julia/` — Stan data preparation, model fitting, post-estimation, enrichment, and validation.
- `models/` — Stan model specification.
- `data/` — five processed party-level inputs used by the Julia/Stan model.
@@ -20,9 +20,9 @@ Processed inputs needed by the model are included in `data/` so the estimation s
The public repository does **not** contain the original raw/source files. Several inputs are third-party datasets with their own terms of use, and the Morgan historical file is a local OCR/transcription source. Instead, this repository provides:
1. committed model-ready inputs in `data/`, sufficient for fitting the Julia/Stan model; and
2. `data-setup/`, a workflow that downloads or checks raw sources and rebuilds comparable model-ready inputs locally.
2. `data-setup/`, an R/Shell workflow that downloads script-accessible sources, checks locally supplied raw sources, and rebuilds comparable model-ready inputs locally.
`data-setup/` documents the required source files and checks locally supplied sources for:
`data-setup/` downloads script-accessible sources and checks locally supplied sources for:
- PolDem
- PartyFacts crosswalk
@@ -44,7 +44,7 @@ Run the full source-data setup workflow with:
bash data-setup/run_data_setup.sh
```
This checks required local files, rebuilds model-ready inputs locally, and writes a comparison report. Manifesto Project requires your own provider access, and the Morgan OCR/transcription file can be provided on request.
This downloads script-accessible source files, checks required local files, rebuilds model-ready inputs locally, and writes a comparison report. Manifesto Project requires your own provider access, and the Morgan OCR/transcription file can be provided on request.
## Running the pipeline
data-setup/R/00_install_dependencies.R
+1 -1
View File
@@ -5,7 +5,7 @@ lib <- Sys.getenv("R_LIBS_USER", file.path(repo_root, "_local", "R", "library"))
dir.create(lib, recursive = TRUE, showWarnings = FALSE)
.libPaths(c(lib, .libPaths()))
required <- c("tidyverse", "countrycode", "haven", "foreign")
required <- c("tidyverse", "countrycode", "haven", "foreign", "jsonlite")
missing <- required[!vapply(required, requireNamespace, quietly = TRUE, FUN.VALUE = logical(1))]
if (length(missing) > 0) {
data-setup/R/01_download_sources.R
+203
View File
@@ -0,0 +1,203 @@
#!/usr/bin/env Rscript
args <- commandArgs(trailingOnly = FALSE)
file_arg <- args[grepl("^--file=", args)][1]
if (!is.na(file_arg)) {
script_path <- sub("^--file=", "", file_arg)
repo_root <- normalizePath(file.path(dirname(script_path), "..", ".."), mustWork = FALSE)
} else {
repo_root <- normalizePath(getwd(), mustWork = TRUE)
}
if (!dir.exists(file.path(repo_root, "data-setup"))) repo_root <- normalizePath(getwd(), mustWork = TRUE)
raw_dir <- normalizePath(Sys.getenv("PARTY2D_RAW_DATA_DIR", file.path(repo_root, "_local", "raw")), mustWork = FALSE)
report_dir <- normalizePath(Sys.getenv("PARTY2D_REPORT_DIR", file.path(repo_root, "_local", "reports")), mustWork = FALSE)
dir.create(raw_dir, recursive = TRUE, showWarnings = FALSE)
dir.create(report_dir, recursive = TRUE, showWarnings = FALSE)
ua <- "party2d-data-setup/1.0 (+https://git.seimel.app/armin/party2d)"
download_file <- function(url, dest, overwrite = FALSE, headers = character()) {
dir.create(dirname(dest), recursive = TRUE, showWarnings = FALSE)
if (file.exists(dest) && file.info(dest)$size > 0 && !overwrite) {
message("OK existing: ", dest)
return(TRUE)
}
tmp <- paste0(dest, ".tmp")
if (file.exists(tmp)) unlink(tmp)
message("Downloading ", url, " -> ", dest)
ok <- tryCatch({
utils::download.file(
url,
tmp,
mode = "wb",
quiet = TRUE,
method = "libcurl",
headers = c("User-Agent" = ua, headers)
)
file.rename(tmp, dest)
}, error = function(e) {
message("FAILED ", url, ": ", conditionMessage(e))
FALSE
})
if (!ok && file.exists(tmp)) unlink(tmp)
isTRUE(ok)
}
download_dataverse <- function(doi, filename, dest, directory = NA_character_) {
if (!requireNamespace("jsonlite", quietly = TRUE)) {
message("FAILED Dataverse lookup: install R package jsonlite")
return(FALSE)
}
api <- paste0("https://dataverse.harvard.edu/api/datasets/:persistentId/?persistentId=", utils::URLencode(doi, reserved = TRUE))
fid <- tryCatch({
meta <- jsonlite::fromJSON(api, simplifyVector = FALSE)
files <- meta$data$latestVersion$files
matches <- Filter(function(x) {
same_file <- identical(x$dataFile$filename, filename)
same_dir <- is.na(directory) || identical(x$directoryLabel, directory)
same_file && same_dir
}, files)
if (length(matches) == 0) NA_integer_ else as.integer(matches[[1]]$dataFile$id)
}, error = function(e) {
message("FAILED Dataverse lookup ", doi, " ", filename, ": ", conditionMessage(e))
NA_integer_
})
if (is.na(fid)) {
message("FAILED Dataverse lookup ", doi, ": file not found: ", filename)
return(FALSE)
}
download_file(paste0("https://dataverse.harvard.edu/api/access/datafile/", fid), dest)
}
download_manifesto <- function(dest) {
key <- Sys.getenv("MANIFESTO_API_KEY", Sys.getenv("PARTY2D_MANIFESTO_API_KEY", ""))
if (!nzchar(key)) {
message("SKIP Manifesto: set MANIFESTO_API_KEY or PARTY2D_MANIFESTO_API_KEY")
return(FALSE)
}
url <- "https://manifesto-project.wzb.eu/api/v1/get_core?key=MPDS2025a&raw=true"
download_file(
url,
dest,
overwrite = TRUE,
headers = c("Referer" = "https://manifesto-project.wzb.eu/datasets", "API_KEY" = key)
)
}
download_vparty <- function(dest) {
email <- Sys.getenv("PARTY2D_VDEM_EMAIL", "")
if (!nzchar(email)) {
message("SKIP V-Party: set PARTY2D_VDEM_EMAIL to use V-Dem's required download form")
return(FALSE)
}
curl <- Sys.which("curl")
if (!nzchar(curl)) {
message("FAILED V-Party: curl is required for the provider form")
return(FALSE)
}
page <- "https://www.v-dem.net/data/v-party-dataset/country-party-date-v2/"
tmpdir <- tempfile("vparty")
dir.create(tmpdir)
on.exit(unlink(tmpdir, recursive = TRUE), add = TRUE)
cookie <- file.path(tmpdir, "cookies.txt")
html <- file.path(tmpdir, "page.html")
payload <- file.path(tmpdir, "payload.bin")
zip_path <- file.path(tmpdir, "vparty.zip")
status <- system2(curl, c("-L", "-A", ua, "-c", cookie, "-b", cookie, "-o", html, page), stdout = TRUE, stderr = TRUE)
if (!file.exists(html)) {
message("FAILED V-Party form load")
return(FALSE)
}
page_text <- paste(readLines(html, warn = FALSE), collapse = "\n")
csrf <- sub('.*name="csrfmiddlewaretoken"[^>]*value="([^"]*)".*', '\\1', page_text)
if (identical(csrf, page_text)) csrf <- ""
gender <- Sys.getenv("PARTY2D_VDEM_GENDER", "")
form <- c(
"csrfmiddlewaretoken", csrf,
"email", email,
"gender", gender,
"accept_terms", "on",
"dataset_file", "17",
"website", ""
)
args <- c(
"-L", "-A", ua, "-c", cookie, "-b", cookie,
"-H", paste0("Referer: ", page),
"-H", paste0("X-CSRFToken: ", csrf),
"-H", "X-Requested-With: XMLHttpRequest",
"-o", payload,
"--data-urlencode", paste0(form[1], "=", form[2]),
"--data-urlencode", paste0(form[3], "=", form[4]),
"--data-urlencode", paste0(form[5], "=", form[6]),
"--data-urlencode", paste0(form[7], "=", form[8]),
"--data-urlencode", paste0(form[9], "=", form[10]),
"--data-urlencode", paste0(form[11], "=", form[12]),
paste0(page, "#dataset-download")
)
system2(curl, args, stdout = TRUE, stderr = TRUE)
if (!file.exists(payload) || file.info(payload)$size == 0) {
message("FAILED V-Party form submit")
return(FALSE)
}
bytes <- readBin(payload, "raw", n = min(file.info(payload)$size, 4))
if (length(bytes) >= 2 && identical(as.integer(bytes[1:2]), c(0x50L, 0x4bL))) {
file.copy(payload, zip_path, overwrite = TRUE)
} else {
text <- paste(readLines(payload, warn = FALSE), collapse = "\n")
m <- regexpr('https?://[^" ]*CPD_V-Party_R_v2\\.zip|/[^" ]*CPD_V-Party_R_v2\\.zip', text)
if (m[1] < 0) {
message("FAILED V-Party form response did not include a recognizable ZIP download")
return(FALSE)
}
url <- regmatches(text, m)[1]
if (startsWith(url, "/")) url <- paste0("https://www.v-dem.net", url)
if (!download_file(url, zip_path, overwrite = TRUE, headers = c("Referer" = page))) return(FALSE)
}
listing <- utils::unzip(zip_path, list = TRUE)
member <- listing$Name[grepl("\\.(rds|rda|rdata)$", listing$Name, ignore.case = TRUE)][1]
if (is.na(member)) {
message("FAILED V-Party ZIP contains no R data file")
return(FALSE)
}
dir.create(dirname(dest), recursive = TRUE, showWarnings = FALSE)
utils::unzip(zip_path, files = member, exdir = tmpdir, overwrite = TRUE)
file.copy(file.path(tmpdir, member), dest, overwrite = TRUE)
message("Extracted V-Party R data: ", dest)
TRUE
}
status <- c()
status["PolDem"] <- download_file("https://poldem.eui.eu/downloads/cosa/poldem-election_all.csv", file.path(raw_dir, "poldem", "poldem-election_all.csv"))
status["PartyFacts external parties"] <- download_file("https://partyfacts.herokuapp.com/download/external-parties-csv/", file.path(raw_dir, "partyfacts", "partyfacts-external-parties.csv"))
status["Manifesto MPDS 2025a"] <- download_manifesto(file.path(raw_dir, "manifesto", "MPDataset_MPDS2025a.csv"))
ches_base <- "https://www.chesdata.eu/s/"
ches_files <- c(
"1999-2019_CHES_dataset_meansv3.csv" = "1999-2019_CHES_dataset_means(v3).csv",
"1999-2024_CHES_dataset_meansV2-3k4l.csv" = "1999-2024_CHES_dataset_meansV2-3k4l.csv",
"CHES_2024_final_v2.csv" = "CHES_2024_final_v2.csv",
"CHES_2024_ALL_Stacked_Expert.csv" = "CHES_2024_expert_level.csv",
"CHES_CA2023.csv" = "CHES_CA2023.csv",
"CHES_CA2023_expert-level.csv" = "CHES_CA2023_expert_level.csv",
"ches_la_2020_aggregate_level_v01.csv" = "ches_la_2020_aggregate_level_v01.csv",
"ches_la_2020_expert_level_v01.csv" = "CHES_LA2020_expert_level.csv",
"CHES_ISRAEL_means_2021_2022.csv" = "CHES_ISRAEL_means_2021_2022.csv",
"CHES_ISRAEL_expert_level_2021_2022.csv" = "CHES_IL_expert_level.csv"
)
for (remote in names(ches_files)) {
local <- ches_files[[remote]]
status[paste("CHES", local)] <- download_file(paste0(ches_base, utils::URLencode(remote, reserved = TRUE)), file.path(raw_dir, "ches", local))
}
status["POPPA integrated v2"] <- download_dataverse("doi:10.7910/DVN/RMQREQ", "poppa_integrated_v2.rds", file.path(raw_dir, "poppa", "poppa_integrated_v2.rds"), "final_data_v2")
status["GPS 2019 party"] <- download_dataverse("doi:10.7910/DVN/WMGTNS", "Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab", file.path(raw_dir, "gps", "Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab"))
status["V-Party"] <- download_vparty(file.path(raw_dir, "vparty", "V-Dem-CPD-Party-V2.rds"))
report <- file.path(report_dir, "download_sources_report.md")
lines <- c("# Download sources report", "", "| source | status |", "| --- | --- |")
for (name in names(status)) lines <- c(lines, paste0("| ", name, " | ", if (isTRUE(status[[name]])) "ok" else "missing/failed", " |"))
writeLines(lines, report)
message("Wrote download report: ", report)
quit(status = if (all(status)) 0 else 2)
data-setup/README.md
+36 -14
View File
@@ -1,6 +1,6 @@
# Data setup
This directory exists because the public repository cannot redistribute the original raw/source files. It checks user-provided source files, rebuilds model-ready inputs, and compares them with the committed files in `../data/`.
This directory exists because the public repository cannot redistribute the original raw/source files. It downloads script-accessible sources, checks user-provided source files, rebuilds model-ready inputs, and compares them with the committed files in `../data/`.
The main estimation workflow does not run these scripts. Once the five model-ready CSVs exist in `data/`, fitting and post-estimation are Julia/Stan-only.
@@ -8,20 +8,20 @@ The setup workflow never overwrites committed files in `data/`.
Put raw source files in `_local/raw/` or set `PARTY2D_RAW_DATA_DIR` to another local directory. See `source_manifest.csv` and `../docs/RAW_DATA_SOURCES.md` for source-specific access notes.
## Source availability
## What downloads automatically?
The public repository does not include an automatic source downloader. Users should obtain source files directly from the providers under the providers' terms and place them under `_local/raw/` or another directory selected with `PARTY2D_RAW_DATA_DIR`.
`data-setup/R/01_download_sources.R` downloads source files that are script-accessible under the providers' terms and reports sources that require credentials or manual local files.
| Source | Requirement |
| --- | --- |
| PolDem | obtain from provider |
| PartyFacts crosswalk | obtain from provider |
| CHES family files | obtain from provider |
| POPPA | obtain from Harvard Dataverse |
| Global Party Survey 2019 | obtain from Harvard Dataverse |
| V-Party | obtain from provider |
| Manifesto Project | use your own Manifesto Project access/API key |
| Morgan historical expert data | place `morgan_positions_raw.csv` locally; available on request |
| Source | Automatic? | Requirement |
| --- | --- | --- |
| PolDem | Yes | none |
| PartyFacts crosswalk | Yes | none |
| CHES family files | Yes | none where provider links are live |
| POPPA | Yes | none; downloaded from Harvard Dataverse |
| Global Party Survey 2019 | Yes | none; downloaded from Harvard Dataverse |
| V-Party | Yes, with provider form details | set `PARTY2D_VDEM_EMAIL`; optionally set `PARTY2D_VDEM_GENDER` |
| Manifesto Project | Yes, with credentials | set your own `MANIFESTO_API_KEY` or `PARTY2D_MANIFESTO_API_KEY` |
| Morgan historical expert data | No | place `morgan_positions_raw.csv` locally; available on request |
Do not commit downloaded or user-provided source files.
@@ -58,9 +58,10 @@ bash data-setup/run_data_setup.sh
Maintenance options:
- `--dry-run` — check setup scripts only.
- `--download-only` — download script-accessible sources.
- `--build-test` — rebuild model-ready inputs from existing local sources.
- `--compare` — compare regenerated inputs with committed `data/` inputs.
- no option — rebuild and compare locally from existing source files.
- no option — download, rebuild, and compare locally.
Minimal check:
@@ -68,6 +69,27 @@ Minimal check:
bash data-setup/run_data_setup.sh --dry-run
```
Download sources:
```bash
bash data-setup/run_data_setup.sh --download-only
```
V-Party:
```bash
export PARTY2D_VDEM_EMAIL='you@example.org'
export PARTY2D_VDEM_GENDER='' # blank means prefer not to say
```
Manifesto Project:
```bash
export MANIFESTO_API_KEY='...'
# or
export PARTY2D_MANIFESTO_API_KEY='...'
```
Morgan is not a public provider download. The local OCR/transcription file can be provided on request and should be placed at:
```text
data-setup/run_data_setup.sh
+13 -1
View File
@@ -7,6 +7,7 @@ Usage: bash data-setup/run_data_setup.sh
Optional maintenance modes:
--dry-run check setup scripts only
--download-only download source files where scriptable
--build-test rebuild inputs from existing local sources
--compare compare regenerated inputs with committed data/
EOF
@@ -22,7 +23,7 @@ repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
cd "$repo_root"
case "$MODE" in
--dry-run|--build-test|--compare|--full-test) ;;
--dry-run|--download-only|--build-test|--compare|--full-test) ;;
*)
usage
exit 1
@@ -51,6 +52,17 @@ if [ "$MODE" = "--compare" ]; then
exit 0
fi
if [ "$MODE" = "--download-only" ]; then
Rscript data-setup/R/00_install_dependencies.R
Rscript data-setup/R/01_download_sources.R
exit 0
fi
if [ "$MODE" = "--full-test" ]; then
Rscript data-setup/R/00_install_dependencies.R
Rscript data-setup/R/01_download_sources.R || true
fi
bash data-setup/check_raw_data.sh
rm -rf "$PARTY2D_BUILD_DIR" "$PARTY2D_GENERATED_INPUT_DIR"
mkdir -p "$PARTY2D_BUILD_DIR" "$PARTY2D_GENERATED_INPUT_DIR"
docs/RAW_DATA_SOURCES.md
+8 -5
View File
@@ -100,14 +100,17 @@ the underlying raw source data.
See `data-setup/source_manifest.csv` for a machine-readable source checklist.
## Source access status
## Scripted download status
The public repository does not include an automatic source downloader. Obtain
source files directly from the providers under their terms and place them in the
documented local raw-data layout before running the rebuild checks.
`data-setup/R/01_download_sources.R` downloads sources that are script-accessible
under provider terms: PolDem, PartyFacts, CHES family files where provider links
are live, POPPA from Harvard Dataverse, GPS from Harvard Dataverse, Manifesto
with user credentials, and V-Party through the provider form when
`PARTY2D_VDEM_EMAIL` is set.
The Manifesto Project main dataset requires Manifesto Project access/API
credentials from the provider.
credentials from the provider. Set `MANIFESTO_API_KEY` or
`PARTY2D_MANIFESTO_API_KEY` for scripted download.
The Morgan historical file is a local OCR/transcription source from Morgan
(1976), not a public provider download. It can be provided on request and must be