Wire local-only data setup workflow

.gitignore

@@ -1,3 +1,4 @@
+# Local-only raw downloads, build intermediates, regenerated inputs, and reports.
 _local/
 tmp/
 outputs/
@@ -5,6 +6,17 @@ outputs/
 data/raw/
 data/staging/
 data/releases/
+data/*.csv
+!data/text_data.csv
+!data/expert.csv
+!data/lr_data.csv
+!data/union_mapping.csv
+!data/party_families.csv
+
+data-setup/_local/
+data-setup/build/
+data-setup/generated-inputs/
+data-setup/reports/
 
 metadata/release_manifest_*.csv
 metadata/scientific_data_release_*.txt

README.md

@@ -4,10 +4,10 @@ Code and processed model inputs for generating two-dimensional party-position es
 
 ## Repository contents
 
-- `src/r/` — scripts that prepare processed model inputs from source datasets.
+- `data-setup/` — optional scripts and source manifest for rebuilding model-ready inputs from local raw files.
 - `src/julia/` — Stan data preparation, model fitting, post-estimation, enrichment, and validation.
 - `models/` — Stan model specification.
-- `data/` — processed party-level inputs used by the model.
+- `data/` — five processed party-level inputs used by the Julia/Stan model.
 - `metadata/` — data dictionary and source-support documentation.
 - `docs/` — raw data source documentation, coding decisions, and operational notes.
 
@@ -21,16 +21,17 @@ Run the full workflow with:
 bash run_estimation.sh full
 ```
 
-This executes the numbered workflow scripts:
+This checks that model-ready inputs are present, then executes the Julia/Stan workflow scripts:
 
 ```bash
-bash scripts/01_prepare_data.sh
+bash scripts/01_prepare_data.sh   # checks model-ready inputs; does not rebuild raw data
 bash scripts/02_fit_model.sh
 bash scripts/03_extract_estimates.sh
 bash scripts/04_enrich_estimates.sh
+bash scripts/05_validate_estimates.sh
 ```
 
-The numbered scripts can also be run manually in that order. `scripts/05_validate_estimates.sh` runs validation checks after estimates have been generated.
+The numbered scripts can also be run manually in that order.
 
 The Bayesian model is computationally expensive. The production run used 4 cores on an AMD Ryzen 9 7945HX and took 60,372 seconds, approximately 16 hours 46 minutes.
 
@@ -40,7 +41,7 @@ If model output is already available, rebuild estimates without refitting Stan:
 bash run_estimation.sh reuse
 ```
 
-`reuse` reruns source-data processing, post-estimation, and enrichment while skipping the Stan fitting step.
+`reuse` verifies the model-ready inputs, then reruns post-estimation, enrichment, and validation while skipping the Stan fitting step.
 
 To check the local setup without fitting the model, run:
 
@@ -50,9 +51,15 @@ bash run_estimation.sh dry-run
 
 ## Data inputs
 
-The model-ready inputs are included under `data/`.
+The model-ready inputs are included under `data/`:
 
-Original raw source files are not redistributed. See `docs/RAW_DATA_SOURCES.md` for the list of original data sources, access information, and expected local filenames for regenerating the processed inputs.
+- `text_data.csv`
+- `expert.csv`
+- `lr_data.csv`
+- `union_mapping.csv`
+- `party_families.csv`
+
+Original raw source files are not redistributed. See `docs/RAW_DATA_SOURCES.md` and `data-setup/` for the list of original data sources, access information, and expected local filenames for regenerating the processed inputs. Rebuilding inputs from raw files is separate from the normal estimation workflow and writes only to ignored `_local/` test directories; it never replaces committed `data/` inputs automatically.
 
 ## Output variables
 

data-setup/R/00_install_dependencies.R

@@ -0,0 +1,16 @@
+#!/usr/bin/env Rscript
+
+repo_root <- normalizePath(getwd(), mustWork = TRUE)
+lib <- Sys.getenv("R_LIBS_USER", file.path(repo_root, "_local", "R", "library"))
+dir.create(lib, recursive = TRUE, showWarnings = FALSE)
+.libPaths(c(lib, .libPaths()))
+
+required <- c("tidyverse", "countrycode", "haven", "foreign")
+missing <- required[!vapply(required, requireNamespace, quietly = TRUE, FUN.VALUE = logical(1))]
+
+if (length(missing) > 0) {
+  message("Installing missing R packages into ", lib, ": ", paste(missing, collapse = ", "))
+  install.packages(missing, repos = "https://cloud.r-project.org", lib = lib)
+} else {
+  message("R data-setup dependencies already available")
+}

data-setup/R/01_download_sources.R

@@ -0,0 +1,5 @@
+#!/usr/bin/env Rscript
+
+script <- file.path("data-setup", "download_sources.py")
+status <- system2("python3", script)
+quit(status = status)

data-setup/R/02_build_model_inputs.R

@@ -0,0 +1,348 @@
+# ============================================================
+# 02_build_model_inputs.R - Master Data Pipeline Orchestrator
+# ============================================================
+# Coordinates all data processing sub-scripts and produces
+# final output files for the 4D latent trait model. By default this writes only
+# to local-only directories under _local/ and never overwrites committed data/.
+#
+# Sub-scripts (run conditionally based on intermediate file existence):
+#   process_manifesto.R  -> manifesto_data.csv
+#   process_poldem.R     -> poldem_data.csv
+#   process_expert.R     -> expert_raw.csv, lr_data_raw.csv
+#   process_morgan.R     -> morgan_data.csv, morgan_lr.csv
+#
+# Final generated model inputs:
+#   text_data.csv - Combined manifesto + PolDem
+#   expert.csv    - Expert survey data (CHES, V-Party, POPPA, GPS)
+#   lr_data.csv   - General left-right anchoring data
+# ============================================================
+
+library(tidyverse)
+library(countrycode)
+
+cmd_args <- commandArgs(trailingOnly = FALSE)
+file_arg <- grep("^--file=", cmd_args, value = TRUE)
+if (length(file_arg) > 0) {
+  this_file <- normalizePath(sub("^--file=", "", file_arg[[1]]), mustWork = TRUE)
+  repo_root <- normalizePath(file.path(dirname(this_file), "..", ".."), mustWork = TRUE)
+} else {
+  repo_root <- normalizePath(getwd(), mustWork = TRUE)
+}
+script_dir <- file.path(repo_root, "data-setup", "R")
+build_dir <- normalizePath(
+  Sys.getenv("PARTY2D_BUILD_DIR", file.path(repo_root, "_local", "build")),
+  mustWork = FALSE
+)
+generated_input_dir <- normalizePath(
+  Sys.getenv("PARTY2D_GENERATED_INPUT_DIR", file.path(repo_root, "_local", "generated-inputs")),
+  mustWork = FALSE
+)
+dir.create(build_dir, recursive = TRUE, showWarnings = FALSE)
+dir.create(generated_input_dir, recursive = TRUE, showWarnings = FALSE)
+
+# The source-processing scripts use relative paths for intermediate files.
+# Keep those intermediates in the ignored build directory, never committed data/.
+setwd(build_dir)
+
+# Static model support inputs are versioned in data/ and copied into the local
+# generated-input set for comparison. They are not regenerated by raw-source setup.
+for (support_file in c("union_mapping.csv", "party_families.csv")) {
+  src <- file.path(repo_root, "data", support_file)
+  if (!file.exists(src)) {
+    stop("Required committed support input not found: ", src)
+  }
+  file.copy(src, file.path(build_dir, support_file), overwrite = TRUE)
+}
+
+cat("============================================================\n")
+cat("Data Management Pipeline\n")
+cat("============================================================\n\n")
+cat("Build directory: ", build_dir, "\n", sep = "")
+cat("Generated input directory: ", generated_input_dir, "\n\n", sep = "")
+
+# ============================================================
+# Configuration: Set to TRUE to force re-run of sub-scripts
+# ============================================================
+
+FORCE_RERUN_MANIFESTO <- FALSE
+FORCE_RERUN_POLDEM <- FALSE
+FORCE_RERUN_EXPERT <- FALSE
+FORCE_RERUN_MORGAN <- FALSE
+
+# ============================================================
+# Step 1: Manifesto Data
+# ============================================================
+
+cat("Step 1: Manifesto data\n")
+if (!file.exists("manifesto_data.csv") || !file.exists("election_data.csv") || FORCE_RERUN_MANIFESTO) {
+  cat("  Running process_manifesto.R...\n")
+  source(file.path(script_dir, "process_manifesto.R"))
+} else {
+  cat("  Loading cached manifesto_data.csv and election_data.csv...\n")
+}
+manifesto <- read_csv("manifesto_data.csv", show_col_types = FALSE)
+election_data <- read_csv("election_data.csv", show_col_types = FALSE)
+cat(sprintf("  Loaded manifesto: %d rows, %d parties\n", nrow(manifesto), n_distinct(manifesto$party)))
+cat(sprintf("  Loaded election: %d rows, %d parties\n\n", nrow(election_data), n_distinct(election_data$party)))
+
+# ============================================================
+# Step 2: PolDem Media Data
+# ============================================================
+
+cat("Step 2: PolDem media data\n")
+if (!file.exists("poldem_data.csv") || FORCE_RERUN_POLDEM) {
+  cat("  Running process_poldem.R...\n")
+  source(file.path(script_dir, "process_poldem.R"))
+} else {
+  cat("  Loading cached poldem_data.csv...\n")
+}
+poldem_data <- read_csv("poldem_data.csv", show_col_types = FALSE)
+cat(sprintf("  Loaded: %d rows, %d parties\n\n", nrow(poldem_data), n_distinct(poldem_data$party)))
+
+# ============================================================
+# Step 4: Expert Survey Data
+# ============================================================
+
+cat("Step 3: Expert survey data\n")
+if (!file.exists("expert_raw.csv") || !file.exists("lr_data_raw.csv") || FORCE_RERUN_EXPERT) {
+  cat("  Running process_expert.R...\n")
+  source(file.path(script_dir, "process_expert.R"))
+} else {
+  cat("  Loading cached expert_raw.csv and lr_data_raw.csv...\n")
+}
+expert_raw <- read_csv("expert_raw.csv", show_col_types = FALSE)
+lr_data_raw <- read_csv("lr_data_raw.csv", show_col_types = FALSE)
+cat(sprintf("  Expert: %d rows, LR: %d rows\n\n", nrow(expert_raw), nrow(lr_data_raw)))
+
+# ============================================================
+# Step 3b: Morgan (1976) Historical Expert Data
+# ============================================================
+
+cat("Step 3b: Morgan (1976) historical L-R data\n")
+
+# First run to generate morgan_data.csv if needed
+if (!file.exists("morgan_data.csv") || FORCE_RERUN_MORGAN) {
+  cat("  Running process_morgan.R (initial processing)...\n")
+  source(file.path(script_dir, "process_morgan.R"))
+}
+
+# morgan_lr.csv depends on text_data.csv, so we need to check if it needs regeneration
+# It will be generated/regenerated below after text_data is created
+
+# ============================================================
+# Step 4: Combine Text Data Sources
+# ============================================================
+
+cat("Step 4: Combining text data sources\n")
+text_data <- bind_rows(manifesto, poldem_data)
+cat(sprintf("  Combined text_data: %d rows\n", nrow(text_data)))
+
+# Save unfiltered text_data for reproducible mismatch diagnosis
+write_csv(text_data, "text_data_unfiltered.csv")
+cat(sprintf("  Saved unfiltered text_data: %d rows, %d parties\n", nrow(text_data), n_distinct(text_data$party)))
+
+# ============================================================
+# Step 4b: Party Renames (applied before filtering)
+# ============================================================
+# Renames must happen BEFORE the relevance filter so that party IDs
+# match across text_data and expert_raw when computing expert coverage.
+
+# Simple renames only (organizational continuity: same leadership/members)
+simple_renames <- c(
+  `10` = 1816L,    # DE: Greens -> Bündnis90/Grüne
+  `276` = 120L,    # RO: FDSN/PDSR -> PSD (renamed 2001)
+  `8054` = 878L,   # IT: PDS -> DS (renamed 1998)
+  `1696` = 813L,   # IT: MSI -> AN (refounded 1995)
+  `553` = 1968L,   # BE: Vlaams Blok -> Vlaams Belang (refounded 2004)
+  `8058` = 1626L   # IT: Forza Italia (refounded 2013) -> Forza Italia (same party, Berlusconi)
+)
+
+apply_simple_renames <- function(df) {
+  for (old_id in names(simple_renames)) {
+    df <- df %>%
+      mutate(party = ifelse(party == as.integer(old_id), simple_renames[[old_id]], party))
+  }
+  df
+}
+
+cat("\nStep 4b: Party renames\n")
+text_data <- apply_simple_renames(text_data)
+cat(sprintf("  Applied %d renames to text_data\n", length(simple_renames)))
+
+# ============================================================
+# Step 4c: Relevance Filter
+# ============================================================
+# Design: R pipeline filters for RELEVANCE (is this party worth modeling?).
+# Julia pipeline handles INTERPOLATION QUALITY (MAX_GAP=7 segment splitting, MIN_OBS=2).
+# Expert survey coverage is a relevance signal: CHES only covers parties with >1% vote share.
+
+cat("\nStep 4c: Relevance filter\n")
+parties_before <- n_distinct(text_data$party)
+
+# Compute expert coverage per party (with renames applied for consistent matching)
+expert_year_counts <- bind_rows(
+  expert_raw %>% select(party, year),
+  lr_data_raw %>% select(party, year)
+) %>% distinct() %>%
+  apply_simple_renames() %>%
+  distinct() %>%
+  count(party, name = "expert_years")
+
+expert_party_ids <- unique(expert_year_counts$party)
+cat(sprintf("  Parties with expert data: %d\n", length(expert_party_ids)))
+
+# Three-tier relevance filter:
+#   Tier 1: 3+ text data years (always include, regardless of expert data)
+#   Tier 2: 2 text years + any expert data (major newer parties like M5S, ANO, LREM)
+#   Tier 3: 1 text year + 3+ expert survey years (parties with rich expert coverage)
+text_data <- text_data %>%
+  group_by(country, party) %>%
+  mutate(n_years = n_distinct(year)) %>%
+  ungroup() %>%
+  left_join(expert_year_counts, by = "party") %>%
+  mutate(expert_years = replace_na(expert_years, 0L)) %>%
+  mutate(
+    tier = case_when(
+      n_years >= 3 ~ 1L,
+      n_years >= 2 & party %in% expert_party_ids ~ 2L,
+      n_years >= 1 & expert_years >= 3 ~ 3L,
+      TRUE ~ 0L
+    )
+  ) %>%
+  filter(tier > 0) %>%
+  select(-n_years, -expert_years, -tier)
+
+parties_after <- n_distinct(text_data$party)
+cat(sprintf("  Parties before filter: %d\n", parties_before))
+cat(sprintf("  Parties after filter: %d\n", parties_after))
+cat(sprintf("  Parties removed: %d\n\n", parties_before - parties_after))
+
+# ============================================================
+# Step 5: Party Harmonization
+# ============================================================
+
+cat("Step 5: Party harmonization (union-aware)\n")
+
+# Load union mapping to identify constituent parties
+union_map <- read_csv("union_mapping.csv", show_col_types = FALSE)
+
+# Build set of constituent parties whose union is in text_data
+constituent_parties <- union_map %>%
+  filter(manifesto_pf_id %in% unique(text_data$party)) %>%
+  pull(expert_pf_id)
+
+cat(sprintf("  Union mappings loaded: %d rows covering %d unions\n",
+            nrow(union_map), n_distinct(union_map$manifesto_pf_id)))
+cat(sprintf("  Constituent parties with unions in text_data: %d\n",
+            length(unique(constituent_parties))))
+
+# Deduplicate union manifesto rows: where multiple CMP codes map to the same
+# union PF ID with identical content, keep only one set per (party, year, var)
+text_data_before_dedup <- nrow(text_data)
+text_data <- text_data %>%
+  distinct(country, party, year, var, .keep_all = TRUE)
+cat(sprintf("  Text data: %d unique parties after harmonization\n", n_distinct(text_data$party)))
+cat(sprintf("  Text data: deduplicated %d -> %d rows\n", text_data_before_dedup, nrow(text_data)))
+
+# Filter expert data: keep parties in text_data OR constituent parties of unions in text_data
+expert <- expert_raw %>%
+  apply_simple_renames() %>%
+  group_by(country, party, var, year) %>%
+  summarise(
+    val = mean(val, na.rm = TRUE),
+    val_int = first(val_int),
+    n_scale = first(n_scale),
+    n_experts = first(n_experts),
+    project = first(project),
+    type_low = first(type_low),
+    type_high = first(type_high),
+    .groups = "drop"
+  ) %>%
+  filter(party %in% unique(text_data$party) | party %in% constituent_parties)
+
+lr_data <- lr_data_raw %>%
+  apply_simple_renames() %>%
+  group_by(country, party, var, year) %>%
+  summarise(
+    val = mean(val, na.rm = TRUE),
+    val_int = first(val_int),
+    n_scale = first(n_scale),
+    n_experts = first(n_experts),
+    project = first(project),
+    .groups = "drop"
+  ) %>%
+  filter(party %in% unique(text_data$party) | party %in% constituent_parties)
+
+cat(sprintf("  Expert data: %d rows (filtered to text_data parties)\n", nrow(expert)))
+cat(sprintf("  LR data (CHES/POPPA): %d rows (filtered to text_data parties)\n", nrow(lr_data)))
+
+# ============================================================
+# Step 5b: Integrate Morgan L-R Data
+# ============================================================
+
+cat("\nStep 5b: Morgan L-R data integration\n")
+
+# Generate morgan_lr.csv (requires text_data.csv to exist)
+# We need to regenerate it if text_data changed or if forced
+if (!file.exists("morgan_lr.csv") || FORCE_RERUN_MORGAN) {
+  cat("  Generating morgan_lr.csv...\n")
+  # Write text_data first so morgan script can use it
+  write_csv(text_data, "text_data.csv")
+  source(file.path(script_dir, "process_morgan.R"))
+}
+
+# Load and integrate Morgan L-R data
+if (file.exists("morgan_lr.csv")) {
+  morgan_lr <- read_csv("morgan_lr.csv", show_col_types = FALSE) %>%
+    apply_simple_renames() %>%
+    filter(party %in% unique(text_data$party) | party %in% constituent_parties)
+
+  cat(sprintf("  Morgan L-R: %d rows (filtered to text_data parties)\n", nrow(morgan_lr)))
+  cat(sprintf("  Morgan parties: %d\n", n_distinct(morgan_lr$party)))
+  cat(sprintf("  Morgan year range: %d-%d\n", min(morgan_lr$year), max(morgan_lr$year)))
+
+  # Combine with existing lr_data
+  lr_data_before <- nrow(lr_data)
+  lr_data <- bind_rows(lr_data, morgan_lr) %>%
+    arrange(country, party, year, var)
+
+  cat(sprintf("  Combined LR data: %d rows (+%d from Morgan)\n",
+              nrow(lr_data), nrow(lr_data) - lr_data_before))
+} else {
+  cat("  Warning: morgan_lr.csv not found, skipping Morgan integration\n")
+}
+
+cat("\n")
+
+# ============================================================
+# Step 6: Write Final Outputs
+# ============================================================
+
+cat("Step 6: Writing final outputs\n")
+
+write_csv(text_data, "text_data.csv")
+write_csv(expert, "expert.csv")
+write_csv(lr_data, "lr_data.csv")
+
+for (final_file in c("text_data.csv", "expert.csv", "lr_data.csv", "union_mapping.csv", "party_families.csv")) {
+  file.copy(file.path(build_dir, final_file), file.path(generated_input_dir, final_file), overwrite = TRUE)
+}
+
+cat("\n============================================================\n")
+cat("Pipeline Complete!\n")
+cat("============================================================\n\n")
+
+cat("Output files written:\n")
+cat(sprintf("  local generated input dir: %s\n", generated_input_dir))
+cat(sprintf("  text_data.csv: %d rows\n", nrow(text_data)))
+cat(sprintf("    - Manifesto: %d rows\n", sum(grepl("_manifesto", text_data$var))))
+cat(sprintf("    - PolDem:    %d rows\n", sum(grepl("_poldem", text_data$var))))
+cat(sprintf("  expert.csv: %d rows\n", nrow(expert)))
+cat(sprintf("  lr_data.csv: %d rows\n", nrow(lr_data)))
+cat(sprintf("    - CHES:   %d rows\n", sum(lr_data$var == "lr_ches")))
+cat(sprintf("    - POPPA:  %d rows\n", sum(lr_data$var == "lr_poppa")))
+cat(sprintf("    - Morgan: %d rows\n", sum(lr_data$var == "lr_morgan")))
+
+cat("\nUnique parties in text_data:", n_distinct(text_data$party), "\n")
+cat("Countries:", paste(sort(unique(text_data$country)), collapse = ", "), "\n")
+cat("Year range:", min(text_data$year, na.rm = TRUE), "-", max(text_data$year, na.rm = TRUE), "\n")

data-setup/R/process_expert.R

@@ -0,0 +1,620 @@
+# ============================================================
+# process_expert.R - Expert Survey Data Processing
+# ============================================================
+# Processes expert survey data from multiple sources:
+# - Chapel Hill Expert Survey (CHES)
+# - V-Party Dataset
+# - POPPA
+# - GPS (Norris)
+#
+# Outputs: expert_raw.csv, lr_data_raw.csv
+#
+# V5 changes:
+# - val_int (integer rounded to nearest scale point) and n_scale columns
+# - n_experts column preserved (not dropped)
+# - V-Party cultural expansion: 5 native items replace GPS ep_v6_lib_cons
+# - V-Party economic expansion: v2pawelf added
+# - Reverse-coding for V-Party cultural + welfare items
+# ============================================================
+
+library(tidyverse)
+library(countrycode)
+library(haven)
+library(foreign)
+
+# Set working directory (works both in RStudio and command line)
+if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
+  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
+}
+
+cat("Processing expert survey data...\n")
+
+raw_data_dir <- Sys.getenv(
+  "PARTY2D_RAW_DATA_DIR",
+  unset = file.path("..", "..", "_local", "raw")
+)
+ches_dir <- file.path(raw_data_dir, "ches")
+vparty_dir <- file.path(raw_data_dir, "vparty")
+poppa_dir <- file.path(raw_data_dir, "poppa")
+gps_dir <- file.path(raw_data_dir, "gps")
+partyfacts_path <- file.path(raw_data_dir, "partyfacts", "partyfacts-external-parties.csv")
+
+ches_country_iso2 <- function(country_id) {
+  lookup <- c(
+    `1` = "BE", `2` = "DK", `3` = "DE", `4` = "GR", `5` = "ES",
+    `6` = "FR", `7` = "IE", `8` = "IT", `10` = "NL", `11` = "GB",
+    `12` = "PT", `13` = "AT", `14` = "FI", `16` = "SE", `20` = "BG",
+    `21` = "CZ", `22` = "EE", `23` = "HU", `24` = "LV", `25` = "LT",
+    `26` = "PL", `27` = "RO", `28` = "SK", `29` = "SI", `31` = "HR",
+    `32` = "TR", `33` = "NO", `34` = "CH", `35` = "MT", `36` = "CY",
+    `37` = "IS", `38` = "CH", `40` = "CY"
+  )
+  unname(lookup[as.character(country_id)])
+}
+
+ches2024_country_iso2 <- function(country_id) {
+  lookup <- c(
+    `1` = "BE", `2` = "DK", `3` = "DE", `4` = "GR", `5` = "ES",
+    `6` = "FR", `7` = "IE", `8` = "IT", `10` = "NL", `11` = "GB",
+    `12` = "PT", `13` = "AT", `14` = "FI", `16` = "SE", `20` = "BG",
+    `21` = "CZ", `22` = "EE", `23` = "HU", `24` = "LV", `25` = "LT",
+    `26` = "PL", `27` = "RO", `28` = "SK", `29` = "SI", `31` = "HR",
+    `34` = "TR", `35` = "NO", `36` = "CH", `37` = "MT", `40` = "CY",
+    `45` = "IS"
+  )
+  unname(lookup[as.character(country_id)])
+}
+
+# ============================================================
+# PartyFacts Linkage for CHES
+# ============================================================
+
+partyfacts_raw <- read_csv(partyfacts_path, show_col_types = FALSE)
+ches_link <- partyfacts_raw %>%
+  filter(dataset_key == "ches") %>%
+  transmute(id = dataset_party_id,
+            country = countrycode(country, origin = 'iso3c', destination = "iso2c"),
+            party = partyfacts_id)
+
+# ============================================================
+# Expert Count Tables (from individual response files)
+# ============================================================
+
+cat("  Loading expert count tables from individual response files...\n")
+
+# CHES 2024: dual lookup (party_id primary, country+name fallback for ID mismatches)
+ches24_exp_raw <- read_csv(file.path(ches_dir, 'CHES_2024_expert_level.csv'), show_col_types = FALSE)
+ches24_exp_by_id <- ches24_exp_raw %>%
+  group_by(party_id) %>%
+  summarise(n_experts_id = as.integer(n_distinct(id)), .groups = "drop")
+ches24_exp_by_name <- ches24_exp_raw %>%
+  mutate(country_iso2 = countrycode(cname, origin = "country.name", destination = "iso2c")) %>%
+  group_by(country_iso2, party_name) %>%
+  summarise(n_experts_name = as.integer(n_distinct(id)), .groups = "drop")
+
+ches_ca_expert_counts <- read_csv(file.path(ches_dir, 'CHES_CA2023_expert_level.csv'), show_col_types = FALSE) %>%
+  group_by(party_id) %>%
+  summarise(n_experts = as.integer(n_distinct(expert)), .groups = "drop")
+
+ches_la_expert_counts <- read_csv(file.path(ches_dir, 'CHES_LA2020_expert_level.csv'), show_col_types = FALSE) %>%
+  group_by(party_id) %>%
+  summarise(n_experts = as.integer(n_distinct(expert_id)), .groups = "drop")
+
+ches_il_expert_counts <- read_csv(file.path(ches_dir, 'CHES_IL_expert_level.csv'), show_col_types = FALSE) %>%
+  group_by(party_id, year) %>%
+  summarise(n_experts = as.integer(n_distinct(id)), .groups = "drop")
+
+# ============================================================
+# Chapel Hill Expert Survey (CHES) - 1999-2019
+# ============================================================
+
+cat("  Processing CHES 1999-2019...\n")
+
+ches <- read_csv(file.path(ches_dir, '1999-2019_CHES_dataset_means(v3).csv'), show_col_types = FALSE) %>%
+  rename(country_id = country) %>%
+  transmute(country = ches_country_iso2(country_id),
+            vote = vote,
+            year = year,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = as.integer(expert),
+            lrecon_ches = lrecon/10,
+            galtan_ches = galtan/10) %>%
+  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
+  mutate(n_scale = 10L) %>%
+  left_join(ches_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id) %>%
+  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
+         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
+
+# ============================================================
+# CHES 2024 Update
+# ============================================================
+
+cat("  Processing CHES 2024...\n")
+
+# Country code lookup for CHES 2024 format
+country_lookup <- c(
+  "be" = "BE", "dk" = "DK", "ge" = "DE", "gr" = "GR", "esp" = "ES",
+  "fr" = "FR", "irl" = "IE", "it" = "IT", "nl" = "NL", "uk" = "GB",
+  "por" = "PT", "aus" = "AT", "fin" = "FI", "sv" = "SE", "bul" = "BG",
+  "cz" = "CZ", "est" = "EE", "hun" = "HU", "lat" = "LV", "lith" = "LT",
+  "pol" = "PL", "rom" = "RO", "slo" = "SK", "sle" = "SI", "cro" = "HR",
+  "tur" = "TR", "nor" = "NO", "swi" = "CH", "mal" = "MT", "cyp" = "CY",
+  "ice" = "IS"
+)
+
+convert_country_codes <- function(codes) {
+  numeric_result <- ches2024_country_iso2(codes)
+  result <- country_lookup[codes]
+  result[is.na(result)] <- numeric_result[is.na(result)]
+  result[is.na(result)] <- codes[is.na(result)]
+  return(unname(result))
+}
+
+ches24 <- read_csv(file.path(ches_dir, 'CHES_2024_final_v2.csv'), show_col_types = FALSE) %>%
+  mutate(country_iso2 = convert_country_codes(country)) %>%
+  left_join(ches24_exp_by_id, by = "party_id") %>%
+  left_join(ches24_exp_by_name, by = c("country_iso2", "party" = "party_name")) %>%
+  transmute(country = country_iso2,
+            vote = vote,
+            year = 2024,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = coalesce(n_experts_id, n_experts_name),
+            lrecon_ches = lrecon/10,
+            galtan_ches = galtan/10) %>%
+  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
+  mutate(n_scale = 10L) %>%
+  left_join(ches_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id) %>%
+  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
+         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
+
+ches <- bind_rows(ches, ches24)
+
+# ============================================================
+# CHES Canada 2023
+# ============================================================
+
+cat("  Processing CHES Canada 2023...\n")
+
+ches_ca <- read_csv(file.path(ches_dir, 'CHES_CA2023.csv'), show_col_types = FALSE) %>%
+  filter(!is.na(partyfacts_id)) %>%
+  left_join(ches_ca_expert_counts, by = "party_id") %>%
+  transmute(country = "CA",
+            year = 2023,
+            party = partyfacts_id,
+            project = 'CHES',
+            n_experts = n_experts,
+            lrecon_ches = lrecon/10,
+            galtan_ches = galtan/10) %>%
+  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
+  mutate(n_scale = 10L) %>%
+  filter(!is.na(val), !is.na(party)) %>%
+  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
+         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
+
+ches <- bind_rows(ches, ches_ca)
+cat(sprintf("    CHES Canada: %d observations\n", nrow(ches_ca)))
+
+# ============================================================
+# CHES Latin America 2020
+# ============================================================
+
+cat("  Processing CHES Latin America 2020...\n")
+
+ches_la_link <- partyfacts_raw %>%
+  filter(dataset_key == "ches") %>%
+  transmute(id = as.character(dataset_party_id),
+            country = countrycode(country, origin = "iso3c", destination = "iso2c"),
+            party = partyfacts_id)
+
+ches_la <- read_csv(file.path(ches_dir, 'ches_la_2020_aggregate_level_v01.csv'), show_col_types = FALSE) %>%
+  left_join(ches_la_expert_counts, by = "party_id") %>%
+  transmute(country = toupper(country_abb),
+            year = 2020,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = n_experts,
+            lrecon_ches = lrecon/10,
+            galtan_ches = galtan/10) %>%
+  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
+  mutate(n_scale = 10L) %>%
+  left_join(ches_la_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id) %>%
+  mutate(type_low = as.character(ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan")),
+         type_high = as.character(ifelse(var == "lrecon_ches", "pro_market", "traditional")))
+
+ches <- bind_rows(ches, ches_la)
+cat(sprintf("    CHES Latin America: %d observations\n", nrow(ches_la)))
+
+# ============================================================
+# CHES Israel 2021-2022
+# ============================================================
+
+cat("  Processing CHES Israel 2021-2022...\n")
+
+ches_il_link <- partyfacts_raw %>%
+  filter(dataset_key == "ches") %>%
+  transmute(id = as.character(dataset_party_id),
+            country = countrycode(country, origin = "iso3c", destination = "iso2c"),
+            party = partyfacts_id)
+
+ches_il <- read_csv(file.path(ches_dir, 'CHES_ISRAEL_means_2021_2022.csv'), show_col_types = FALSE) %>%
+  left_join(ches_il_expert_counts, by = c("party_id", "year")) %>%
+  transmute(country = "IL",
+            year = year,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = n_experts,
+            lrecon_ches = lrecon/10,
+            galtan_ches = galtan/10) %>%
+  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
+  mutate(n_scale = 10L) %>%
+  left_join(ches_il_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id) %>%
+  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
+         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
+
+ches <- bind_rows(ches, ches_il)
+cat(sprintf("    CHES Israel: %d observations\n", nrow(ches_il)))
+
+cat(sprintf("    CHES total: %d observations\n", nrow(ches)))
+
+# ============================================================
+# CHES General Left-Right (for anchoring)
+# ============================================================
+
+cat("  Processing CHES LR anchoring data...\n")
+
+ches_lr <- read_csv(file.path(ches_dir, '1999-2019_CHES_dataset_means(v3).csv'), show_col_types = FALSE) %>%
+  rename(country_id = country) %>%
+  transmute(country = ches_country_iso2(country_id),
+            vote = vote,
+            year = year,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = as.integer(expert),
+            val = lrgen/10,
+            var = 'lr_ches',
+            n_scale = 10L) %>%
+  left_join(ches_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id)
+
+ches24_lr <- read_csv(file.path(ches_dir, 'CHES_2024_final_v2.csv'), show_col_types = FALSE) %>%
+  mutate(country_iso2 = convert_country_codes(country)) %>%
+  left_join(ches24_exp_by_id, by = "party_id") %>%
+  left_join(ches24_exp_by_name, by = c("country_iso2", "party" = "party_name")) %>%
+  transmute(country = country_iso2,
+            vote = vote,
+            year = 2024,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = coalesce(n_experts_id, n_experts_name),
+            val = lrgen/10,
+            var = 'lr_ches',
+            n_scale = 10L) %>%
+  left_join(ches_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id)
+
+# CHES Canada LR
+ches_ca_lr <- read_csv(file.path(ches_dir, 'CHES_CA2023.csv'), show_col_types = FALSE) %>%
+  filter(!is.na(partyfacts_id)) %>%
+  left_join(ches_ca_expert_counts, by = "party_id") %>%
+  transmute(country = "CA",
+            year = 2023,
+            party = partyfacts_id,
+            project = 'CHES',
+            n_experts = n_experts,
+            val = lrgen/10,
+            var = 'lr_ches',
+            n_scale = 10L) %>%
+  filter(!is.na(val), !is.na(party))
+
+# CHES Latin America LR
+ches_la_lr <- read_csv(file.path(ches_dir, 'ches_la_2020_aggregate_level_v01.csv'), show_col_types = FALSE) %>%
+  left_join(ches_la_expert_counts, by = "party_id") %>%
+  transmute(country = toupper(country_abb),
+            year = 2020,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = n_experts,
+            val = lrgen/10,
+            var = 'lr_ches',
+            n_scale = 10L) %>%
+  left_join(ches_la_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id)
+
+# CHES Israel LR
+ches_il_lr <- read_csv(file.path(ches_dir, 'CHES_ISRAEL_means_2021_2022.csv'), show_col_types = FALSE) %>%
+  left_join(ches_il_expert_counts, by = c("party_id", "year")) %>%
+  transmute(country = "IL",
+            year = year,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = n_experts,
+            val = lrgen/10,
+            var = 'lr_ches',
+            n_scale = 10L) %>%
+  left_join(ches_il_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id)
+
+ches_lr <- bind_rows(ches_lr, ches24_lr, ches_ca_lr, ches_la_lr, ches_il_lr)
+
+# ============================================================
+# V-Party Dataset (V5: expanded to 7 variables)
+# ============================================================
+
+cat("  Processing V-Party...\n")
+
+vparty_raw <- readRDS(file.path(vparty_dir, 'V-Dem-CPD-Party-V2.rds'))
+
+# Economic 1: v2pariglef_osp (0-6 scale, higher = more right, NO reverse)
+vparty_econ1 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2pariglef_nr),
+    val = v2pariglef_osp / 6,
+    val_int = as.integer(round(v2pariglef_osp)),
+    n_scale = 6L,
+    var = "lrecon_vparty",
+    type_low = "pro_welfare",
+    type_high = "pro_market"
+  ) %>%
+  na.omit()
+
+# Economic 2 (NEW): v2pawelf_osp (0-5 scale, higher = more welfare = LEFT, REVERSE)
+vparty_econ2 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2pawelf_nr),
+    val = 1 - v2pawelf_osp / 5,
+    val_int = 5L - as.integer(round(v2pawelf_osp)),
+    n_scale = 5L,
+    var = "welf_vparty",
+    type_low = "pro_welfare",
+    type_high = "pro_market"
+  ) %>%
+  na.omit()
+
+# Cultural 1 (NEW): v2paimmig_osp (0-4 scale, higher = more pro-immigration = GAL, REVERSE)
+vparty_cult1 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2paimmig_nr),
+    val = 1 - v2paimmig_osp / 4,
+    val_int = 4L - as.integer(round(v2paimmig_osp)),
+    n_scale = 4L,
+    var = "immig_vparty",
+    type_low = "cosmopolitan",
+    type_high = "traditional"
+  ) %>%
+  na.omit()
+
+# Cultural 2 (NEW): v2palgbt_osp (0-4 scale, higher = more pro-LGBT = GAL, REVERSE)
+vparty_cult2 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2palgbt_nr),
+    val = 1 - v2palgbt_osp / 4,
+    val_int = 4L - as.integer(round(v2palgbt_osp)),
+    n_scale = 4L,
+    var = "lgbt_vparty",
+    type_low = "cosmopolitan",
+    type_high = "traditional"
+  ) %>%
+  na.omit()
+
+# Cultural 3 (NEW): v2paculsup_osp (0-4 scale, higher = less cultural superiority = GAL, REVERSE)
+vparty_cult3 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2paculsup_nr),
+    val = 1 - v2paculsup_osp / 4,
+    val_int = 4L - as.integer(round(v2paculsup_osp)),
+    n_scale = 4L,
+    var = "culsup_vparty",
+    type_low = "cosmopolitan",
+    type_high = "traditional"
+  ) %>%
+  na.omit()
+
+# Cultural 4 (NEW): v2parelig_osp (0-4 scale, higher = less religious = GAL, REVERSE)
+vparty_cult4 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2parelig_nr),
+    val = 1 - v2parelig_osp / 4,
+    val_int = 4L - as.integer(round(v2parelig_osp)),
+    n_scale = 4L,
+    var = "relig_vparty",
+    type_low = "cosmopolitan",
+    type_high = "traditional"
+  ) %>%
+  na.omit()
+
+# Cultural 5 (NEW): v2pagender_osp (0-4 scale, higher = more pro-gender equality = GAL, REVERSE)
+vparty_cult5 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2pagender_nr),
+    val = 1 - v2pagender_osp / 4,
+    val_int = 4L - as.integer(round(v2pagender_osp)),
+    n_scale = 4L,
+    var = "gender_vparty",
+    type_low = "cosmopolitan",
+    type_high = "traditional"
+  ) %>%
+  na.omit()
+
+vparty <- bind_rows(vparty_econ1, vparty_econ2,
+                    vparty_cult1, vparty_cult2, vparty_cult3,
+                    vparty_cult4, vparty_cult5)
+
+cat(sprintf("    V-Party: %d observations (7 variables)\n", nrow(vparty)))
+cat(sprintf("      lrecon: %d, welf: %d\n", nrow(vparty_econ1), nrow(vparty_econ2)))
+cat(sprintf("      immig: %d, lgbt: %d, culsup: %d, relig: %d, gender: %d\n",
+            nrow(vparty_cult1), nrow(vparty_cult2), nrow(vparty_cult3),
+            nrow(vparty_cult4), nrow(vparty_cult5)))
+
+# ============================================================
+# POPPA Dataset
+# ============================================================
+
+cat("  Processing POPPA...\n")
+
+poppa <- readRDS(file.path(poppa_dir, 'poppa_integrated_v2.rds')) %>%
+  transmute(country = countrycode(country_short, origin = "iso3c", destination = "iso2c"),
+            party = partyfacts_id,
+            val = lrecon/10,
+            var = "lrecon_poppa",
+            type_low = "pro_welfare",
+            type_high = "pro_market",
+            n_experts = as.integer(n_experts),
+            n_scale = 10L,
+            year = as.numeric(sub(".*-\\s*(\\d+)", "\\1", wave)),
+            project = "POPPA") %>%
+  na.omit()
+
+cat(sprintf("    POPPA: %d observations\n", nrow(poppa)))
+
+# POPPA General LR
+poppa_lr <- readRDS(file.path(poppa_dir, 'poppa_integrated_v2.rds')) %>%
+  transmute(country = countrycode(country_short, origin = "iso3c", destination = "iso2c"),
+            party = partyfacts_id,
+            val = lroverall/10,
+            var = "lr_poppa",
+            n_experts = as.integer(n_experts),
+            n_scale = 10L,
+            year = as.numeric(sub(".*-\\s*(\\d+)", "\\1", wave)),
+            project = "POPPA") %>%
+  na.omit()
+
+# ============================================================
+# GPS (Norris) Survey
+# ============================================================
+
+cat("  Processing GPS...\n")
+
+gps <- read.delim(file.path(gps_dir, "Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab")) %>%
+  transmute(n_experts = as.integer(Experts),
+            lrecon_gps = as.numeric(V4_Scale)/10,
+            libcon_gps = as.numeric(V6_Scale)/10,
+            party = ID_PartyFacts,
+            country = countrycode(ifelse(ISO == "MAC", "MKD", ISO), origin = "iso3c", destination = "iso2c"),
+            year = 2019,
+            n_scale = 10L,
+            project = "GPS") %>%
+  pivot_longer(cols = lrecon_gps:libcon_gps, names_to = 'var', values_to = 'val') %>%
+  mutate(type_low = ifelse(var == "lrecon_gps", "pro_welfare", "cosmopolitan"),
+         type_high = ifelse(var == "lrecon_gps", "pro_market", "traditional")) %>%
+  na.omit()
+
+cat(sprintf("    GPS: %d observations\n", nrow(gps)))
+
+# ============================================================
+# Combine Expert Data
+# ============================================================
+
+cat("  Combining expert surveys...\n")
+
+expert_raw <- select(ches, -vote) %>%
+  bind_rows(vparty) %>%
+  bind_rows(gps) %>%
+  bind_rows(poppa) %>%
+  unique() %>%
+  arrange(country, party, year, var) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country), !is.na(var))
+
+# Compute val_int for datasets that don't have it pre-computed
+# V-Party already has val_int; CHES/GPS/POPPA need it computed from val * n_scale
+expert_raw <- expert_raw %>%
+  mutate(
+    val_int = ifelse(is.na(val_int), as.integer(round(val * n_scale)), val_int),
+    val_int = pmin(pmax(val_int, 0L), n_scale)
+  )
+
+# Boundary adjustments for continuous val (avoid exact 0 or 1 for Stan prior means)
+expert_raw <- expert_raw %>%
+  mutate(
+    val = case_when(
+      val == 0 ~ val + 1e-4,
+      val == 1 ~ val - 1e-4,
+      TRUE ~ val
+    ))
+
+# ============================================================
+# Combine LR Data
+# ============================================================
+
+lr_data_raw <- ches_lr %>%
+  bind_rows(poppa_lr) %>%
+  select(-any_of("vote"))
+
+# Boundary adjustments for continuous val (avoid exact 0 or 1)
+lr_data_raw <- lr_data_raw %>%
+  mutate(
+    val = case_when(
+      val == 0 ~ val + 1e-4,
+      val == 1 ~ val - 1e-4,
+      TRUE ~ val
+    ))
+
+# Compute val_int for LR data
+lr_data_raw <- lr_data_raw %>%
+  mutate(
+    val_int = as.integer(round(val * n_scale)),
+    val_int = pmin(pmax(val_int, 0L), n_scale)
+  )
+
+# ============================================================
+# Write Outputs
+# ============================================================
+
+write_csv(expert_raw, "expert_raw.csv")
+write_csv(lr_data_raw, "lr_data_raw.csv")
+
+cat(sprintf("\nOutputs written:\n"))
+cat(sprintf("  expert_raw.csv: %d rows\n", nrow(expert_raw)))
+cat(sprintf("  lr_data_raw.csv: %d rows\n", nrow(lr_data_raw)))
+
+cat("\n  Expert data by source:\n")
+expert_raw %>%
+  group_by(project) %>%
+  summarise(n = n(), .groups = "drop") %>%
+  print()
+
+cat("\n  New columns check:\n")
+cat(sprintf("  val_int range: %d - %d\n", min(expert_raw$val_int), max(expert_raw$val_int)))
+cat(sprintf("  n_scale values: %s\n", paste(sort(unique(expert_raw$n_scale)), collapse = ", ")))
+cat(sprintf("  n_experts non-NA: %d / %d\n", sum(!is.na(expert_raw$n_experts)), nrow(expert_raw)))

data-setup/R/process_manifesto.R

@@ -0,0 +1,179 @@
+# ============================================================
+# process_manifesto.R - Manifesto Project Data Processing
+# ============================================================
+# Processes Manifesto Project data for the 4D latent trait model
+# Input: $PARTY2D_RAW_DATA_DIR/manifesto/MPDataset_MPDS2025a.csv
+# Output: manifesto_data.csv
+# ============================================================
+
+library(tidyverse)
+library(countrycode)
+library(purrr)
+
+# Set working directory (works both in RStudio and command line)
+if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
+  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
+}
+
+cat("Processing Manifesto Project data...\n")
+
+raw_data_dir <- Sys.getenv(
+  "PARTY2D_RAW_DATA_DIR",
+  unset = file.path("..", "..", "_local", "raw")
+)
+manifesto_raw_path <- file.path(raw_data_dir, "manifesto", "MPDataset_MPDS2025a.csv")
+partyfacts_path <- file.path(raw_data_dir, "partyfacts", "partyfacts-external-parties.csv")
+
+# ============================================================
+# PartyFacts Linkage
+# ============================================================
+
+partyfacts_raw <- read_csv(partyfacts_path, show_col_types = FALSE)
+manifesto_link <- partyfacts_raw %>%
+  filter(dataset_key == "manifesto") %>%
+  transmute(id = dataset_party_id,
+            country = countrycode(country, origin = 'iso3c', destination = "iso2c"),
+            party = partyfacts_id,
+            party = ifelse(party == 622, 604, party))
+
+# ============================================================
+# Load Manifesto Data
+# ============================================================
+
+manifesto_data <- read_csv(manifesto_raw_path, show_col_types = FALSE)
+
+# ============================================================
+# CMP Code Mapping to 4 Dimensions
+# ============================================================
+
+vars <- tribble(
+  ~type,          ~subtype,                 ~per_var,  ~stance,    ~label,
+  # pro_market
+  "pro_market",   "Market Regulation",      "per401",  "Positive", "Free Market Economy",
+  "pro_market",   "Economic Liberalization","per402",  "Positive", "Incentives: Positive",
+  "pro_market",   "Market Regulation",      "per407",  "Positive", "Protectionism: Negative",
+  "pro_market",   "Economic Liberalization","per414",  "Positive", "Economic Orthodoxy",
+  "pro_market",   "Economic Liberalization","per505",  "Positive", "Welfare State Limitation",
+  "pro_market",   "Economic Liberalization","per507",  "Positive", "Education Limitation",
+  "pro_market",   "Economic Liberalization","per702",  "Positive", "Labour Groups: Negative",
+  "pro_market",   "Market Regulation",      "per406",  "Negative", "Protectionism: Positive",
+  "pro_market",   "Market Regulation",      "per412",  "Negative", "Controlled Economy",
+  "pro_market",   "Economic Liberalization","per504",  "Negative", "Welfare State Expansion",
+  # pro_welfare
+  "pro_welfare",  "Economic Intervention",  "per403",  "Positive", "Market Regulation",
+  "pro_welfare",  "Economic Intervention",  "per404",  "Positive", "Economic Planning",
+  "pro_welfare",  "Economic Intervention",  "per412",  "Positive", "Controlled Economy",
+  "pro_welfare",  "Economic Intervention",  "per413",  "Positive", "Nationalisation",
+  "pro_welfare",  "Social Services",        "per504",  "Positive", "Welfare State Expansion",
+  "pro_welfare",  "Social Services",        "per506",  "Positive", "Education Expansion",
+  "pro_welfare",  "Economic Intervention",  "per701",  "Positive", "Labour Groups: Positive",
+  "pro_welfare",  "Economic Intervention",  "per401",  "Negative", "Free Market Economy",
+  "pro_welfare",  "Social Services",        "per505",  "Negative", "Welfare State Limitation",
+  # cosmopolitan
+  "cosmopolitan", "Internationalism",       "per107",  "Positive", "Internationalism: Positive",
+  "cosmopolitan", "Internationalism",       "per108",  "Positive", "European Community/Union: Positive",
+  "cosmopolitan", "Multiculturalism",       "per607",  "Positive", "Multiculturalism: Positive",
+  "cosmopolitan", "Multiculturalism",       "per201",  "Positive", "Freedom and Human Rights",
+  "cosmopolitan", "Multiculturalism",       "per604",  "Positive", "traditional Morality: Negative",
+  "cosmopolitan", "Internationalism",       "per109",  "Negative", "Internationalism: Negative",
+  "cosmopolitan", "Multiculturalism",       "per601",  "Negative", "National Way of Life: Positive",
+  # traditional
+  "traditional",  "National Identity",      "per109",  "Positive", "Internationalism: Negative",
+  "traditional",  "Conservative Morality",  "per110",  "Positive", "European Community/Union: Negative",
+  "traditional",  "National Identity",      "per601",  "Positive", "National Way of Life: Positive",
+  "traditional",  "Conservative Morality",  "per603",  "Positive", "traditional Morality: Positive",
+  "traditional",  "Conservative Morality",  "per608",  "Positive", "Multiculturalism: Negative",
+  "traditional",  "Conservative Morality",  "per605",  "Positive", "Law and Order: Positive",
+  "traditional",  "National Identity",      "per107",  "Negative", "Internationalism: Positive",
+  "traditional",  "Conservative Morality",  "per607",  "Negative", "Multiculturalism: Positive"
+)
+
+# ============================================================
+# Process Manifesto Data
+# ============================================================
+
+manifesto <- vars %>%
+  pmap_dfr(~ manifesto_data %>%
+             transmute(country = countrycode(countryname, origin = 'country.name', destination = 'iso2c'),
+                       year = as.numeric(format(as.Date(edate, format = "%d/%m/%Y"), "%Y")),
+                       id = as.character(party),
+                       count = round(.data[[..3]]),
+                       var = ..3,
+                       label = ..5,
+                       type = ..1,
+                       subtype = ..2,
+                       stance = ..4,
+                       project = 'Manifesto Project') %>%
+             left_join(manifesto_link, by = c("id", "country")) %>%
+             select(-id)) %>%
+  group_by(party, country, year, subtype) %>%
+  summarise(
+    positive = sum(count[stance == "Positive"], na.rm = TRUE),
+    sample = sum(count, na.rm = TRUE),
+    type = first(type),
+    project = first(project),
+    .groups = "drop"
+  ) %>%
+  na.omit() %>%
+  rename(var = subtype) %>%
+  # Convert to bipolar bridge structure (type_high/type_low)
+  mutate(
+    type_high = case_when(
+      type == "pro_welfare" ~ "pro_welfare",
+      type == "pro_market" ~ "pro_market",
+      type == "cosmopolitan" ~ "cosmopolitan",
+      type == "traditional" ~ "traditional"
+    ),
+    type_low = case_when(
+      type %in% c("pro_welfare", "pro_market") ~ ifelse(type == "pro_welfare", "pro_market", "pro_welfare"),
+      type %in% c("cosmopolitan", "traditional") ~ ifelse(type == "cosmopolitan", "traditional", "cosmopolitan")
+    )
+  ) %>%
+  select(-type) %>%
+  # Add _manifesto suffix to variable names
+  mutate(var = paste0(tolower(gsub(" ", "_", var)), "_manifesto"))
+
+# ============================================================
+# NOTE: Temporal continuity filter moved to the data setup orchestrator.
+# This allows exempting parties that appear in parliamentary data
+# (parties in parliament are by definition not fringe parties)
+# ============================================================
+
+cat("Skipping temporal filter (applied in 02_build_model_inputs.R after combining with other text data)\n")
+cat(sprintf("  Parties: %d\n", n_distinct(manifesto$party)))
+
+# ============================================================
+# Write Output
+# ============================================================
+
+write_csv(manifesto, "manifesto_data.csv")
+cat(sprintf("Output: manifesto_data.csv (%d rows, %d parties)\n",
+            nrow(manifesto), n_distinct(manifesto$party)))
+
+# ============================================================
+# Election Data Extraction (vote shares)
+# ============================================================
+
+cat("\nExtracting election data (pervote)...\n")
+
+election_data <- manifesto_data %>%
+  transmute(
+    country = countrycode(countryname, origin = 'country.name', destination = 'iso2c'),
+    year = as.numeric(format(as.Date(edate, format = "%d/%m/%Y"), "%Y")),
+    id = as.character(party),
+    pervote = pervote
+  ) %>%
+  left_join(manifesto_link, by = c("id", "country")) %>%
+  select(-id) %>%
+  filter(!is.na(party), !is.na(pervote)) %>%
+  # Keep one row per (party, country, year) — take max pervote if duplicates
+  group_by(party, country, year) %>%
+  summarise(pervote = max(pervote, na.rm = TRUE), .groups = "drop") %>%
+  arrange(country, party, year)
+
+write_csv(election_data, "election_data.csv")
+cat(sprintf("Output: election_data.csv (%d rows, %d parties)\n",
+            nrow(election_data), n_distinct(election_data$party)))
+
+# Export manifesto_link for use by other scripts
+# (poldem also needs it for CMP linkage)

data-setup/R/process_morgan.R

@@ -0,0 +1,399 @@
+# process_morgan.R
+# Process Morgan (1976) expert party position data
+#
+# Source: Morgan, Michael-John (1976). "The Modelling of Governmental
+# Coalition Formation: A Policy-Based Approach with Interval Measurement."
+# PhD dissertation, University of Michigan.
+#
+# Data extracted from Appendix B.3 (Tables B.3.1-B.3.12) via OCR.
+# Position scores are 25%-truncated means (midmeans) from expert surveys.
+# Scale: 0-100 (left-right)
+
+library(tidyverse)
+
+cat("Processing Morgan (1976) expert party position data...\n")
+
+raw_data_dir <- Sys.getenv(
+  "PARTY2D_RAW_DATA_DIR",
+  unset = file.path("..", "..", "_local", "raw")
+)
+morgan_raw_path <- file.path(raw_data_dir, "morgan", "morgan_positions_raw.csv")
+partyfacts_path <- file.path(raw_data_dir, "partyfacts", "partyfacts-external-parties.csv")
+
+# Load raw extracted data
+morgan_raw <- read_csv(morgan_raw_path, show_col_types = FALSE)
+
+cat(sprintf("Loaded %d party-period observations from %d countries\n",
+            nrow(morgan_raw), n_distinct(morgan_raw$country)))
+
+# Load PartyFacts linkage data
+partyfacts <- read_csv(partyfacts_path, show_col_types = FALSE)
+
+# Filter to Morgan dataset entries
+morgan_pf <- partyfacts %>%
+  filter(dataset_key == "morgan") %>%
+  select(country, name_short, name_english, year_first, year_last,
+         external_id, partyfacts_id) %>%
+  rename(party_abbrev_pf = name_short)
+
+cat(sprintf("Found %d Morgan parties in PartyFacts\n", nrow(morgan_pf)))
+
+# Map extracted abbreviations to PartyFacts abbreviations
+# Some adjustments needed due to OCR/transcription differences
+abbrev_map <- tribble(
+  ~country, ~party_abbrev, ~party_abbrev_pf,
+  # Denmark
+  "DNK", "SOCd", "SOCD",
+  "DNK", "SOCL", "SOCL",
+  "DNK", "COMM", "COMM",
+  "DNK", "RAD", "RAD",
+  "DNK", "LIB", "LIB",
+  "DNK", "CONS", "CONS",
+  "DNK", "LS", "LS",
+  "DNK", "LC", "LC",
+  "DNK", "JUST", "JUST",
+  # Finland
+  "FIN", "SKDL", "SKDL",
+  "FIN", "SOCd", "SOCD",
+  "FIN", "PROG", "PROG",
+  "FIN", "AGR", "AGR",
+  "FIN", "SWPP", "SWPP",
+  "FIN", "CONS", "CONS",
+  "FIN", "NPF", "NPF",
+  "FIN", "PDEM", "PDEM",
+  "FIN", "SDWS", "SDWS",
+  "FIN", "CENT", "CENT",
+  "FIN", "FRP", "FRP",
+  "FIN", "LIB", "LIB",
+  # Iceland
+  "ISL", "COMM", "COMM",
+  "ISL", "SOCd", "SOCD",
+  "ISL", "PROG", "PROG",
+  "ISL", "LIB", "LIB",
+  "ISL", "INDP", "INDP",
+  "ISL", "CONS", "CONS",
+  "ISL", "LLIB", "LLIB",
+  # Norway
+  "NOR", "LAB", "LAB",
+  "NOR", "LIB", "LIB",
+  "NOR", "AGR", "AGR",
+  "NOR", "CONS", "CONS",
+  "NOR", "COMM", "COMM",
+  "NOR", "SOCL", "SOCL",
+  "NOR", "CHPP", "CHPP",
+  "NOR", "CENT", "CENT",
+  # Sweden
+  "SWE", "COMM", "COMM",
+  "SWE", "SOCd", "SOCD",
+  "SWE", "AGR", "AGR",
+  "SWE", "LIB", "LIB",
+  "SWE", "CONS", "CONS",
+  "SWE", "CENT", "CENT",
+  # Netherlands
+  "NLD", "CPN", "CPN",
+  "NLD", "SOCd", "SOCD",
+  "NLD", "RAD", "RAD",
+  "NLD", "KVP", "KVP",
+  "NLD", "CHU", "CHU",
+  "NLD", "LIB", "LIB",
+  "NLD", "ARP", "ARP",
+  "NLD", "SGP", "SGP",
+  "NLD", "NSB", "NSB",
+  "NLD", "PVDA", "PVDA",
+  "NLD", "VVD", "VVD",
+  "NLD", "PSP", "PSP",
+  "NLD", "PPR", "PPR",
+  "NLD", "D66", "D66",
+  "NLD", "DS70", "DS70",
+  "NLD", "GPV", "GPV",
+  "NLD", "BP", "BP",
+  # Belgium
+  "BEL", "COMM", "COMM",
+  "BEL", "POB", "POB",
+  "BEL", "CATH", "CATH",
+  "BEL", "LIB", "LIB",
+  "BEL", "FNAT", "FNAT",
+  "BEL", "REX", "REX",
+  "BEL", "PSB", "PSB",
+  "BEL", "RW", "RW",
+  "BEL", "PSC", "PSC",
+  "BEL", "FDF", "FDF",
+  "BEL", "VOLK", "VOLK",
+  "BEL", "PLP", "PLP",
+  # France (Fourth Republic)
+  "FRA", "PCF", "PCF",
+  "FRA", "SFIO", "SFIO",
+  "FRA", "MRP", "MRP",
+  "FRA", "RDA", "RDA",
+  "FRA", "UDSR", "UDSR",
+  "FRA", "RAD", "RAD",
+  "FRA", "RS", "RS",
+  "FRA", "RPF", "RPF",
+  "FRA", "AR", "AR",
+  "FRA", "ARS", "ARS",
+  "FRA", "RI", "RI",
+  "FRA", "CNIP", "CNIP",
+  "FRA", "PUS", "PUS",
+  "FRA", "PAYS", "PAYS",
+  "FRA", "AP", "AP",
+  "FRA", "PRL", "PRL",
+  "FRA", "POUJ", "POUJ",
+  # Weimar Germany
+  "DEU", "KPD", "KPD",
+  "DEU", "SDAP", "SDAP",
+  "DEU", "DDP", "DDP",
+  "DEU", "DZP", "DZP",
+  "DEU", "BVP", "BVP",
+  "DEU", "DVP", "DVP",
+  "DEU", "RDMW", "RDMW",
+  "DEU", "LVP", "LVP",
+  "DEU", "DNVP", "DNVP",
+  "DEU", "NAZI", "NAZI",
+  # Italy
+  "ITA", "PCI", "PCI",
+  "ITA", "PSIU", "PSIU",
+  "ITA", "PSI", "PSI",
+  "ITA", "PSDI", "PSDI",
+  "ITA", "PRI", "PRI",
+  "ITA", "DC", "DC",
+  "ITA", "PLI", "PLI",
+  "ITA", "MON", "MON",
+  "ITA", "MSI", "MSI",
+  # Luxembourg
+  "LUX", "COMM", "COMM",
+  "LUX", "SOCd", "SOCD",
+  "LUX", "CSOC", "CSOC",
+  "LUX", "GRPD", "GRPD",
+  # Israel
+  "ISR", "RAKA", "RAKA",
+  "ISR", "MAKI", "MAKI",
+  "ISR", "MAPM", "MAPM",
+  "ISR", "MADT", "MADT",
+  "ISR", "ADUT", "ADUT",
+  "ISR", "MAAR", "MAAR",
+  "ISR", "LAB", "LAB",
+  "ISR", "MAPI", "MAPI",
+  "ISR", "PAUG", "PAUG",
+  "ISR", "RAFI", "RAFI",
+  "ISR", "PROG", "PROG",
+  "ISR", "ILIB", "ILIB",
+  "ISR", "NRP", "NRP",
+  "ISR", "URF", "URF",
+  "ISR", "LIB", "LIB",
+  "ISR", "NATL", "NATL",
+  "ISR", "TORA", "TORA",
+  "ISR", "LIKD", "LIKD",
+  "ISR", "ZION", "ZION",
+  "ISR", "GHAL", "GHAL",
+  "ISR", "AGDT", "AGDT",
+  "ISR", "HRUT", "HRUT"
+)
+
+# Some parties in raw data that don't have exact matches - need special handling
+# (e.g., parties that only exist in one period in PartyFacts but appear in both)
+# We'll join using the period-based matching
+
+# Expand periods to years for matching
+morgan_expanded <- morgan_raw %>%
+  mutate(
+    year_start = as.integer(str_extract(period, "^\\d{4}")),
+    year_end = as.integer(str_extract(period, "\\d{4}$"))
+  )
+
+# Join with abbreviation map
+morgan_mapped <- morgan_expanded %>%
+  left_join(abbrev_map, by = c("country", "party_abbrev"))
+
+# Check for unmatched abbreviations
+unmatched_abbrev <- morgan_mapped %>%
+  filter(is.na(party_abbrev_pf)) %>%
+  distinct(country, party_abbrev)
+
+if (nrow(unmatched_abbrev) > 0) {
+  cat("\nWarning: Unmatched abbreviations:\n")
+  print(unmatched_abbrev)
+}
+
+# Join with PartyFacts
+morgan_joined <- morgan_mapped %>%
+  left_join(morgan_pf, by = c("country", "party_abbrev_pf")) %>%
+  # For parties with overlapping periods, use period overlap
+  mutate(
+    period_overlap = pmax(0,
+      pmin(year_end, year_last) - pmax(year_start, year_first) + 1)
+  ) %>%
+  # Keep best match per party-period (max overlap)
+  group_by(country, party_abbrev, period) %>%
+  slice_max(period_overlap, n = 1, with_ties = FALSE) %>%
+  ungroup()
+
+# Check for unmatched parties
+unmatched <- morgan_joined %>%
+  filter(is.na(partyfacts_id)) %>%
+  distinct(country, party_abbrev, party_name, period)
+
+if (nrow(unmatched) > 0) {
+  cat(sprintf("\n%d party-periods without PartyFacts match:\n", nrow(unmatched)))
+  print(unmatched)
+}
+
+# Dedup: when multiple abbreviations map to the same PF ID, keep only one
+matched <- morgan_joined %>%
+  filter(!is.na(partyfacts_id)) %>%
+  group_by(country, partyfacts_id, period) %>%
+  slice(1) %>%
+  ungroup()
+
+cat(sprintf("\nMatched %d of %d party-period observations (%.1f%%)\n",
+            nrow(matched), nrow(morgan_raw),
+            100 * nrow(matched) / nrow(morgan_raw)))
+
+# Normalize position to [0,1] scale
+# Original scale: 0-100
+# Apply boundary adjustments like other expert data
+eps <- 0.005
+morgan_processed <- matched %>%
+  mutate(
+    # Normalize to [0,1]
+    lr_morgan = position / 100,
+    # Apply boundary adjustments
+    lr_morgan = case_when(
+      lr_morgan <= 0 ~ eps,
+      lr_morgan >= 1 ~ 1 - eps,
+      TRUE ~ lr_morgan
+    ),
+    # Calculate standard error (sd / sqrt(n))
+    lr_morgan_se = (sd / 100) / sqrt(n_surveys),
+    # Set minimum SE for extreme parties (sd=0)
+    lr_morgan_se = pmax(lr_morgan_se, 0.01)
+  ) %>%
+  select(
+    country,
+    partyfacts_id,
+    period,
+    year_start,
+    year_end,
+    party_abbrev,
+    party_name,
+    lr_morgan,
+    lr_morgan_se,
+    n_surveys
+  ) %>%
+  arrange(country, year_start, lr_morgan)
+
+# Summary statistics
+cat("\nSummary of processed Morgan data:\n")
+cat(sprintf("  Countries: %d\n", n_distinct(morgan_processed$country)))
+cat(sprintf("  Parties: %d\n", n_distinct(morgan_processed$partyfacts_id)))
+cat(sprintf("  Observations: %d\n", nrow(morgan_processed)))
+
+# Distribution of positions
+cat("\nPosition distribution:\n")
+print(summary(morgan_processed$lr_morgan))
+
+# Write output
+write_csv(morgan_processed, "morgan_data.csv")
+cat(sprintf("\nWrote morgan_data.csv with %d rows\n", nrow(morgan_processed)))
+
+# Also provide a summary by country and period
+summary_by_country <- morgan_processed %>%
+  group_by(country, period) %>%
+  summarise(
+    n_parties = n(),
+    mean_pos = mean(lr_morgan),
+    sd_pos = sd(lr_morgan),
+    .groups = "drop"
+  )
+
+cat("\nSummary by country and period:\n")
+print(summary_by_country, n = 50)
+
+# ============================================================
+# Generate lr_data-compatible output for pipeline integration
+# ============================================================
+
+cat("\n============================================================\n")
+cat("Generating lr_data-compatible output (postwar only)\n")
+cat("============================================================\n")
+
+# Load text_data to get party-years with manifesto/PolDem coverage
+if (!file.exists("text_data.csv")) {
+  cat("text_data.csv not present yet; skipping morgan_lr.csv generation on this pass.\n")
+} else {
+text_data <- read_csv("text_data.csv", show_col_types = FALSE)
+
+# Convert Morgan ISO3 country codes to ISO2 (matching text_data format)
+iso3_to_iso2 <- c(
+  "DNK" = "DK", "FIN" = "FI", "ISL" = "IS", "NOR" = "NO", "SWE" = "SE",
+  "NLD" = "NL", "BEL" = "BE", "DEU" = "DE", "FRA" = "FR", "ITA" = "IT",
+  "LUX" = "LU", "ISR" = "IL"
+)
+
+# Filter to postwar periods only (1945+)
+morgan_postwar <- morgan_processed %>%
+  filter(year_end >= 1945) %>%
+  mutate(country_iso2 = iso3_to_iso2[country])
+
+cat(sprintf("Postwar Morgan observations: %d party-periods\n", nrow(morgan_postwar)))
+cat(sprintf("Countries: %s\n", paste(unique(morgan_postwar$country_iso2), collapse = ", ")))
+
+# Get unique party-years from text_data
+text_party_years <- text_data %>%
+  select(party, country, year) %>%
+  distinct()
+
+cat(sprintf("Unique party-years in text_data: %d\n", nrow(text_party_years)))
+
+# For each Morgan party-period, expand to all years where that party has text data
+# within the Morgan period range (1945-1973 for postwar)
+morgan_lr <- morgan_postwar %>%
+  # Join with text_data party-years
+  # Many-to-many is expected: one Morgan party-period maps to multiple years
+  inner_join(
+    text_party_years,
+    by = c("partyfacts_id" = "party", "country_iso2" = "country"),
+    relationship = "many-to-many"
+  ) %>%
+  # Keep only years within the Morgan period
+  filter(year >= year_start & year <= year_end) %>%
+  # Format for lr_data.csv compatibility
+  transmute(
+    country = country_iso2,
+    party = partyfacts_id,
+    var = "lr_morgan",
+    year = year,
+    val = lr_morgan,
+    project = "Morgan",
+    # Morgan's continuous 0-100 scale is discretized to 10 points (matching CHES
+    # resolution) with the actual number of experts. The reconstructed sum
+    # round(mean × K × 10) is analogous to how CHES means are handled.
+    # See docs/k_scaling_validation.md Section 4.
+    n_scale = 10L,
+    val_int = as.integer(round(lr_morgan * 10)),
+    n_experts = as.integer(n_surveys)
+  ) %>%
+  distinct() %>%
+  arrange(country, party, year)
+
+cat(sprintf("\nGenerated %d lr_morgan observations\n", nrow(morgan_lr)))
+cat(sprintf("  Unique parties: %d\n", n_distinct(morgan_lr$party)))
+cat(sprintf("  Year range: %d-%d\n", min(morgan_lr$year), max(morgan_lr$year)))
+
+# Summary by country
+morgan_lr_summary <- morgan_lr %>%
+  group_by(country) %>%
+  summarise(
+    n_parties = n_distinct(party),
+    n_obs = n(),
+    year_min = min(year),
+    year_max = max(year),
+    .groups = "drop"
+  )
+
+cat("\nMorgan L-R data by country:\n")
+print(morgan_lr_summary, n = 20)
+
+# Write morgan_lr.csv
+write_csv(morgan_lr, "morgan_lr.csv")
+cat(sprintf("\nWrote morgan_lr.csv with %d rows\n", nrow(morgan_lr)))
+}

data-setup/R/process_poldem.R

@@ -0,0 +1,162 @@
+# ============================================================
+# process_poldem.R - PolDem Media Data Processing
+# ============================================================
+# Processes PolDem (Political Deliberation in the Media) data
+# for the 4D latent trait model
+#
+# Input: $PARTY2D_RAW_DATA_DIR/poldem/poldem-election_all.csv (sentence-level)
+# Output: poldem_data.csv (party-year-var aggregates)
+# ============================================================
+
+library(tidyverse)
+library(countrycode)
+
+# Set working directory (works both in RStudio and command line)
+if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
+  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
+}
+
+cat("Processing PolDem media data...\n")
+
+raw_data_dir <- Sys.getenv(
+  "PARTY2D_RAW_DATA_DIR",
+  unset = file.path("..", "..", "_local", "raw")
+)
+poldem_raw_path <- file.path(raw_data_dir, "poldem", "poldem-election_all.csv")
+partyfacts_path <- file.path(raw_data_dir, "partyfacts", "partyfacts-external-parties.csv")
+
+# ============================================================
+# PartyFacts Linkage (via CMP party IDs)
+# ============================================================
+
+partyfacts_raw <- read_csv(partyfacts_path, show_col_types = FALSE)
+manifesto_link <- partyfacts_raw %>%
+  filter(dataset_key == "manifesto") %>%
+  transmute(cmp = as.numeric(dataset_party_id),  # Convert to numeric for join
+            country_pf = countrycode(country, origin = 'iso3c', destination = "iso2c"),
+            party = partyfacts_id,
+            party = ifelse(party == 622, 604, party))
+
+# ============================================================
+# Issue Category Mapping to 4 Dimensions
+# ============================================================
+# For positive direction: type_high is the active trait
+# For negative direction: we flip (same data, just contributes to the opposite trait)
+
+poldem_mapping <- tribble(
+  ~issue_cat,     ~dimension,  ~type_high,      ~type_low,
+  # Economic dimension
+  "ecolib",       "economic",  "pro_market",    "pro_welfare",   # Economic liberalization
+  "welfare",      "economic",  "pro_welfare",   "pro_market",    # Welfare state
+  # Final exclusion: the PolDem economic-reform category is intentionally
+  # omitted because item-response diagnostics showed that it did not load
+  # substantively onto the economic latent trait.
+  # Cultural dimension
+  "immig",        "cultural",  "cosmopolitan",  "traditional",   # Immigration (pro = cosmopolitan)
+  "cultlib",      "cultural",  "cosmopolitan",  "traditional",   # Cultural liberalism
+  "nationalism",  "cultural",  "traditional",   "cosmopolitan",  # Nationalism (pro = traditional)
+  "europe",       "cultural",  "cosmopolitan",  "traditional",   # EU integration (pro = cosmopolitan)
+  "euro",         "cultural",  "cosmopolitan",  "traditional",   # Euro currency (pro = cosmopolitan)
+  "defense",      "cultural",  "traditional",   "cosmopolitan",  # Defense (pro = traditional)
+  "security",     "cultural",  "traditional",   "cosmopolitan"   # Security/law-order (pro = traditional)
+)
+
+cat(sprintf("  Using %d issue categories\n", nrow(poldem_mapping)))
+
+# ============================================================
+# Load and Clean PolDem Data
+# ============================================================
+
+poldem_raw <- read_csv(poldem_raw_path, show_col_types = FALSE)
+cat(sprintf("  Raw PolDem data: %d rows\n", nrow(poldem_raw)))
+
+poldem <- poldem_raw %>%
+  # Fix country codes
+  mutate(country = case_when(
+    iso2code == "AU" ~ "AT",  # Austria (PolDem uses AU instead of AT)
+    iso2code == "UK" ~ "GB",  # United Kingdom
+    TRUE ~ iso2code
+  )) %>%
+  # Extract year from article date (format: YYYY-MM-DD)
+  mutate(year = suppressWarnings(as.numeric(substr(date_art, 1, 4)))) %>%
+  # Filter to valid issue categories only
+  filter(issue_cat %in% poldem_mapping$issue_cat) %>%
+  # Convert direction to numeric and filter out neutral/NA
+  mutate(direction = as.numeric(direction)) %>%
+  filter(!is.na(direction) & direction != 0) %>%
+  # Remove rows with invalid years
+  filter(!is.na(year))
+
+cat(sprintf("  After filtering: %d rows (valid issues, non-neutral)\n", nrow(poldem)))
+
+# ============================================================
+# Link to PartyFacts via CMP codes
+# ============================================================
+
+poldem <- poldem %>%
+  mutate(cmp = as.numeric(cmp)) %>%
+  left_join(manifesto_link, by = "cmp") %>%
+  filter(!is.na(party))
+
+# Report linkage
+n_linked <- nrow(poldem)
+n_unlinked <- nrow(poldem_raw %>%
+                     filter(issue_cat %in% poldem_mapping$issue_cat) %>%
+                     mutate(direction = as.numeric(direction)) %>%
+                     filter(!is.na(direction) & direction != 0)) - n_linked
+
+cat(sprintf("  Linked to PartyFacts: %d rows\n", n_linked))
+if (n_unlinked > 0) {
+  cat(sprintf("  Warning: %d rows could not be linked (missing CMP mapping)\n", n_unlinked))
+}
+
+# ============================================================
+# Aggregate to Party-Year-Issue Level
+# Using round(sum()) for weak direction values (0.5, -0.5)
+# ============================================================
+
+poldem_agg <- poldem %>%
+  left_join(poldem_mapping, by = "issue_cat") %>%
+  group_by(party, country, year, issue_cat, type_high, type_low) %>%
+  summarise(
+    # Sum positive directions (0.5 and 1), then round
+    positive = round(sum(direction[direction > 0])),
+    # Sum absolute directions for sample (all non-neutral), then round
+    sample = round(sum(abs(direction))),
+    n_obs = n(),
+    .groups = "drop"
+  ) %>%
+  # Minimum 3 observations per group
+  filter(n_obs >= 3) %>%
+  select(-n_obs)
+
+cat(sprintf("  After aggregation: %d party-year-issue observations\n", nrow(poldem_agg)))
+
+# ============================================================
+# Format Output (matching manifesto structure)
+# ============================================================
+
+poldem_data <- poldem_agg %>%
+  mutate(
+    var = paste0(issue_cat, "_poldem"),
+    project = "PolDem"
+  ) %>%
+  select(party, country, year, var, positive, sample, type_high, type_low, project)
+
+# ============================================================
+# Write Output
+# ============================================================
+
+write_csv(poldem_data, "poldem_data.csv")
+
+cat(sprintf("\nOutput: poldem_data.csv\n"))
+cat(sprintf("  Total rows: %d\n", nrow(poldem_data)))
+cat(sprintf("  Unique parties: %d\n", n_distinct(poldem_data$party)))
+cat(sprintf("  Countries: %s\n", paste(sort(unique(poldem_data$country)), collapse = ", ")))
+cat(sprintf("  Year range: %d-%d\n", min(poldem_data$year, na.rm = TRUE), max(poldem_data$year, na.rm = TRUE)))
+cat("\n  Rows by issue category:\n")
+poldem_data %>%
+  group_by(var) %>%
+  summarise(n = n(), .groups = "drop") %>%
+  arrange(desc(n)) %>%
+  print()

data-setup/README.md

@@ -0,0 +1,83 @@
+# Data setup
+
+This directory documents and scripts the optional process for rebuilding the model-ready inputs from local raw/source datasets. The main estimation workflow does not run these scripts; once the five model-ready CSVs exist in `data/`, fitting and post-estimation are Julia/Stan-only.
+
+The setup workflow is intentionally local-only. It never overwrites committed files in `data/`. Raw downloads, intermediate files, regenerated inputs, and comparison reports go under `_local/`, which is ignored by git.
+
+Raw source files are not redistributed in this repository. Put them in `_local/raw/` or set `PARTY2D_RAW_DATA_DIR` to another local directory. See `source_manifest.csv` and `../docs/RAW_DATA_SOURCES.md` for source-specific access notes.
+
+Recommended local layout:
+
+```text
+_local/raw/
+  manifesto/MPDataset_MPDS2025a.csv
+  poldem/poldem-election_all.csv
+  ches/...
+  vparty/...
+  poppa/...
+  gps/...
+  morgan/...
+  partyfacts/partyfacts-external-parties.csv
+```
+
+Local output layout:
+
+```text
+_local/build/             # intermediate processing files
+_local/generated-inputs/  # regenerated final model-input CSVs
+_local/reports/           # comparison reports
+```
+
+Check setup without downloading or rebuilding:
+
+```bash
+bash data-setup/run_data_setup.sh --dry-run
+```
+
+Attempt automatic downloads where permitted/available:
+
+```bash
+bash data-setup/run_data_setup.sh --download-only
+```
+
+The downloader handles public direct downloads, Harvard Dataverse files, and the
+V-Dem form workflow. For V-Dem, set an email address accepted by the provider's
+download form:
+
+```bash
+export PARTY2D_VDEM_EMAIL='you@example.org'
+export PARTY2D_VDEM_GENDER=''   # blank means prefer not to say
+```
+
+The Manifesto Project main dataset is behind the Manifesto API/login terms. To
+download it through the script, set one of:
+
+```bash
+export MANIFESTO_API_KEY='...'
+# or
+export PARTY2D_MANIFESTO_API_KEY='...'
+```
+
+`morgan/morgan_positions_raw.csv` is a local OCR/transcription source derived
+from Morgan (1976), not a public provider download. Place it under
+`_local/raw/morgan/` before a full rebuild test.
+
+Rebuild model-ready inputs after placing all required local files. This writes to `_local/generated-inputs/`, not `data/`:
+
+```bash
+bash data-setup/run_data_setup.sh --build-test
+```
+
+Compare regenerated inputs with the committed inputs:
+
+```bash
+bash data-setup/run_data_setup.sh --compare
+```
+
+Run the full local test sequence — public downloads where available, raw-file preflight, local rebuild, and comparison — after manually placing restricted sources:
+
+```bash
+bash data-setup/run_data_setup.sh --full-test
+```
+
+The comparison writes `_local/reports/input_comparison.md` and exits nonzero if any generated input is missing or differs. Replacing committed inputs, if ever needed, is a separate manual decision and is not done by these scripts.

data-setup/check_raw_data.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+raw_data_dir="${PARTY2D_RAW_DATA_DIR:-$repo_root/_local/raw}"
+report_dir="${PARTY2D_REPORT_DIR:-$repo_root/_local/reports}"
+mkdir -p "$report_dir"
+missing_report="$report_dir/raw_data_preflight_missing.txt"
+: > "$missing_report"
+
+required_files=(
+  "manifesto/MPDataset_MPDS2025a.csv"
+  "poldem/poldem-election_all.csv"
+  "partyfacts/partyfacts-external-parties.csv"
+  "ches/1999-2019_CHES_dataset_means(v3).csv"
+  "ches/CHES_2024_final_v2.csv"
+  "ches/CHES_2024_expert_level.csv"
+  "ches/CHES_CA2023.csv"
+  "ches/CHES_CA2023_expert_level.csv"
+  "ches/ches_la_2020_aggregate_level_v01.csv"
+  "ches/CHES_LA2020_expert_level.csv"
+  "ches/CHES_ISRAEL_means_2021_2022.csv"
+  "ches/CHES_IL_expert_level.csv"
+  "vparty/V-Dem-CPD-Party-V2.rds"
+  "poppa/poppa_integrated_v2.rds"
+  "gps/Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab"
+  "morgan/morgan_positions_raw.csv"
+)
+
+echo "Raw data directory: $raw_data_dir"
+echo
+echo "Required raw inputs for regeneration:"
+
+missing=0
+for rel in "${required_files[@]}"; do
+  path="$raw_data_dir/$rel"
+  if [ -s "$path" ]; then
+    bytes="$(wc -c < "$path")"
+    read -r sha _ < <(sha256sum "$path")
+    echo "  OK       $rel ($bytes bytes, sha256=$sha)"
+  else
+    echo "  MISSING  $rel"
+    printf '%s\n' "$rel" >> "$missing_report"
+    missing=1
+  fi
+done
+
+if [ "$missing" -ne 0 ]; then
+  echo
+  echo "At least one required raw input is missing." >&2
+  echo "Missing-file report: $missing_report" >&2
+  echo "See data-setup/README.md and docs/RAW_DATA_SOURCES.md for instructions." >&2
+  exit 1
+fi
+
+echo
+echo "Required raw data preflight passed."
+rm -f "$missing_report"

data-setup/compare_generated_inputs.py

@@ -0,0 +1,121 @@
+#!/usr/bin/env python3
+"""Compare local regenerated model inputs with committed inputs.
+
+This script is intentionally read-only with respect to data/. It writes a report
+under _local/reports/ and exits nonzero when files differ or are missing.
+"""
+
+from __future__ import annotations
+
+import csv
+import hashlib
+import os
+from pathlib import Path
+from typing import Iterable
+
+
+MODEL_INPUTS = [
+    "text_data.csv",
+    "expert.csv",
+    "lr_data.csv",
+    "union_mapping.csv",
+    "party_families.csv",
+]
+
+
+def normalized_rows(path: Path) -> Iterable[list[str]]:
+    with path.open(newline="", encoding="utf-8") as fh:
+        reader = csv.reader(fh)
+        for row in reader:
+            yield ["" if value is None else value.strip() for value in row]
+
+
+def normalized_hash(path: Path) -> str:
+    h = hashlib.sha256()
+    for row in normalized_rows(path):
+        h.update(("\x1f".join(row) + "\n").encode("utf-8"))
+    return h.hexdigest()
+
+
+def csv_shape(path: Path) -> tuple[int, int, list[str]]:
+    with path.open(newline="", encoding="utf-8") as fh:
+        reader = csv.reader(fh)
+        try:
+            header = next(reader)
+        except StopIteration:
+            return 0, 0, []
+        rows = sum(1 for _ in reader)
+        return rows, len(header), header
+
+
+def first_difference(left: Path, right: Path, max_scan: int = 100000) -> str:
+    for idx, (lrow, rrow) in enumerate(zip(normalized_rows(left), normalized_rows(right)), start=1):
+        if lrow != rrow:
+            return f"line {idx}: committed={lrow[:8]!r} generated={rrow[:8]!r}"
+        if idx >= max_scan:
+            break
+    return "no differing row found in scan window; row counts may differ"
+
+
+def main() -> int:
+    repo_root = Path(__file__).resolve().parents[1]
+    committed_dir = Path(os.environ.get("PARTY2D_COMMITTED_INPUT_DIR", repo_root / "data"))
+    generated_dir = Path(os.environ.get("PARTY2D_GENERATED_INPUT_DIR", repo_root / "_local" / "generated-inputs"))
+    report_dir = Path(os.environ.get("PARTY2D_REPORT_DIR", repo_root / "_local" / "reports"))
+    report_dir.mkdir(parents=True, exist_ok=True)
+    report_path = report_dir / "input_comparison.md"
+
+    lines: list[str] = []
+    lines.append("# Generated input comparison")
+    lines.append("")
+    lines.append(f"Committed input dir: `{committed_dir}`")
+    lines.append(f"Generated input dir: `{generated_dir}`")
+    lines.append("")
+    lines.append("| file | status | committed rows | generated rows | committed hash | generated hash | notes |")
+    lines.append("| --- | --- | ---: | ---: | --- | --- | --- |")
+
+    ok = True
+    for name in MODEL_INPUTS:
+        committed = committed_dir / name
+        generated = generated_dir / name
+        if not committed.exists() or not generated.exists():
+            ok = False
+            lines.append(
+                f"| `{name}` | missing |  |  |  |  | committed exists={committed.exists()}, generated exists={generated.exists()} |"
+            )
+            continue
+
+        c_rows, c_cols, c_header = csv_shape(committed)
+        g_rows, g_cols, g_header = csv_shape(generated)
+        c_hash = normalized_hash(committed)
+        g_hash = normalized_hash(generated)
+        same = c_hash == g_hash
+        status = "match" if same else "diff"
+        if not same:
+            ok = False
+        notes = []
+        if c_cols != g_cols:
+            notes.append(f"columns {c_cols}!={g_cols}")
+        if c_header != g_header:
+            notes.append("header differs")
+        if c_rows != g_rows:
+            notes.append(f"rows {c_rows}!={g_rows}")
+        if not same and not notes:
+            notes.append(first_difference(committed, generated))
+        lines.append(
+            f"| `{name}` | {status} | {c_rows} | {g_rows} | `{c_hash[:12]}` | `{g_hash[:12]}` | {'; '.join(notes)} |"
+        )
+
+    lines.append("")
+    lines.append("Committed inputs were not modified by this comparison.")
+    report_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print(f"Wrote comparison report: {report_path}")
+    if ok:
+        print("Generated inputs match committed inputs.")
+        return 0
+    print("Generated inputs differ from committed inputs or are missing.")
+    return 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

data-setup/download_sources.py

@@ -0,0 +1,240 @@
+#!/usr/bin/env python3
+"""Download all script-accessible raw sources into _local/raw/.
+
+The downloader keeps every file inside PARTY2D_RAW_DATA_DIR (default:
+_local/raw). It does not write to committed data/. Sources that require an API
+key or provider form are handled explicitly and reported rather than silently
+skipped.
+"""
+
+from __future__ import annotations
+
+import html.parser
+import json
+import http.cookiejar
+import os
+import re
+import shutil
+import sys
+import urllib.parse
+import urllib.request
+import zipfile
+from pathlib import Path
+
+
+UA = "party2d-data-setup/1.0 (+https://git.seimel.app/armin/party2d)"
+
+
+def raw_dir() -> Path:
+    return Path(os.environ.get("PARTY2D_RAW_DATA_DIR", Path.cwd() / "_local" / "raw")).resolve()
+
+
+def report_dir() -> Path:
+    return Path(os.environ.get("PARTY2D_REPORT_DIR", Path.cwd() / "_local" / "reports")).resolve()
+
+
+def request(url: str, data: bytes | None = None, headers: dict[str, str] | None = None):
+    h = {"User-Agent": UA}
+    if headers:
+        h.update(headers)
+    return urllib.request.Request(url, data=data, headers=h)
+
+
+def download(url: str, dest: Path, *, overwrite: bool = False, headers: dict[str, str] | None = None) -> bool:
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    if dest.exists() and dest.stat().st_size > 0 and not overwrite:
+        print(f"OK existing: {dest}")
+        return True
+    tmp = dest.with_suffix(dest.suffix + ".tmp")
+    print(f"Downloading {url} -> {dest}")
+    try:
+        with urllib.request.urlopen(request(url, headers=headers), timeout=180) as r, tmp.open("wb") as fh:
+            shutil.copyfileobj(r, fh)
+        tmp.replace(dest)
+        return True
+    except Exception as e:
+        if tmp.exists():
+            tmp.unlink()
+        print(f"FAILED {url}: {type(e).__name__}: {e}")
+        return False
+
+
+def dataverse_file_id(doi: str, filename: str, directory: str | None = None) -> int | None:
+    api = "https://dataverse.harvard.edu/api/datasets/:persistentId/?persistentId=" + urllib.parse.quote(doi, safe="")
+    with urllib.request.urlopen(request(api), timeout=120) as r:
+        data = json.load(r)
+    for f in data["data"]["latestVersion"]["files"]:
+        df = f["dataFile"]
+        if df["filename"] == filename and (directory is None or f.get("directoryLabel") == directory):
+            return int(df["id"])
+    return None
+
+
+def download_dataverse(doi: str, filename: str, dest: Path, directory: str | None = None) -> bool:
+    try:
+        fid = dataverse_file_id(doi, filename, directory)
+    except Exception as e:
+        print(f"FAILED Dataverse lookup {doi} {filename}: {type(e).__name__}: {e}")
+        return False
+    if fid is None:
+        print(f"FAILED Dataverse lookup {doi}: file not found: {directory or ''}/{filename}")
+        return False
+    return download(f"https://dataverse.harvard.edu/api/access/datafile/{fid}", dest)
+
+
+class InputParser(html.parser.HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.inputs: dict[str, str] = {}
+
+    def handle_starttag(self, tag, attrs):
+        if tag != "input":
+            return
+        d = dict(attrs)
+        name = d.get("name")
+        if name:
+            self.inputs[name] = d.get("value", "")
+
+
+def download_vparty(dest: Path) -> bool:
+    """Download V-Party R zip through V-Dem's required form.
+
+    V-Dem requires email, gender, privacy acceptance, and format selection. We do
+    not invent personal info: set PARTY2D_VDEM_EMAIL and PARTY2D_VDEM_GENDER
+    (woman/man/non_binary/empty for prefer-not-to-say) to enable this download.
+    """
+    email = os.environ.get("PARTY2D_VDEM_EMAIL")
+    if not email:
+        print("SKIP V-Party: set PARTY2D_VDEM_EMAIL to use V-Dem's required download form")
+        return False
+    gender = os.environ.get("PARTY2D_VDEM_GENDER", "")
+    page = "https://www.v-dem.net/data/v-party-dataset/country-party-date-v2/"
+    cj = http.cookiejar.CookieJar()
+    opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
+    try:
+        with opener.open(request(page), timeout=60) as r:
+            html = r.read().decode("utf-8", "replace")
+    except Exception as e:
+        print(f"FAILED V-Party form load: {type(e).__name__}: {e}")
+        return False
+    p = InputParser(); p.feed(html)
+    csrf = p.inputs.get("csrfmiddlewaretoken", "")
+    fields = {
+        "csrfmiddlewaretoken": csrf,
+        "email": email,
+        "gender": gender,
+        "accept_terms": "on",
+        "dataset_file": "17",  # R format in current V-Dem form
+        "website": "",
+    }
+    data = urllib.parse.urlencode(fields).encode()
+    headers = {
+        "Content-Type": "application/x-www-form-urlencoded",
+        "Referer": page,
+        "X-CSRFToken": csrf,
+        "X-Requested-With": "XMLHttpRequest",
+    }
+    dest.parent.mkdir(parents=True, exist_ok=True)
+    tmp = dest.with_suffix(".zip")
+    print(f"Submitting V-Dem form -> {tmp}")
+    try:
+        with opener.open(request(page + "#dataset-download", data=data, headers=headers), timeout=180) as r:
+            payload = r.read()
+            ctype = r.headers.get("content-type", "")
+        # The AJAX form may return JSON with a download URL, or the file itself.
+        if b"download" in payload[:1000] and "json" in ctype:
+            js = json.loads(payload.decode("utf-8"))
+            url = js.get("download_url") or js.get("url") or js.get("file")
+            if url and url.startswith("/"):
+                url = urllib.parse.urljoin(page, url)
+            if not url:
+                print(f"FAILED V-Party form response did not include download URL: {js}")
+                return False
+            if not download(url, tmp, overwrite=True, headers={"Referer": page}):
+                return False
+        else:
+            text = payload.decode("utf-8", "replace")
+            m = re.search(r'href="([^"]*CPD_V-Party_R_v2\.zip[^"]*)"', text)
+            if m:
+                url = urllib.parse.urljoin(page, m.group(1))
+                if not download(url, tmp, overwrite=True, headers={"Referer": page}):
+                    return False
+            else:
+                tmp.write_bytes(payload)
+        if zipfile.is_zipfile(tmp):
+            with zipfile.ZipFile(tmp) as z:
+                r_files = [n for n in z.namelist() if n.lower().endswith((".rds", ".rda", ".rdata"))]
+                if not r_files:
+                    print("FAILED V-Party zip contains no R data file")
+                    return False
+                member = r_files[0]
+                with z.open(member) as src, dest.open("wb") as out:
+                    shutil.copyfileobj(src, out)
+            print(f"Extracted V-Party R data: {dest}")
+            return True
+        print("FAILED V-Party download was not a zip file")
+        return False
+    except Exception as e:
+        print(f"FAILED V-Party form submit: {type(e).__name__}: {e}")
+        return False
+
+
+def download_manifesto(dest: Path) -> bool:
+    key = os.environ.get("MANIFESTO_API_KEY") or os.environ.get("PARTY2D_MANIFESTO_API_KEY")
+    if not key:
+        print("SKIP Manifesto: set MANIFESTO_API_KEY or PARTY2D_MANIFESTO_API_KEY")
+        return False
+    url = "https://manifesto-project.wzb.eu/api/v1/get_core?" + urllib.parse.urlencode({"key": "MPDS2025a", "raw": "true"})
+    return download(
+        url,
+        dest,
+        overwrite=True,
+        headers={"Referer": "https://manifesto-project.wzb.eu/datasets", "API_KEY": key},
+    )
+
+
+def write_report(status: dict[str, bool]) -> None:
+    rd = report_dir(); rd.mkdir(parents=True, exist_ok=True)
+    path = rd / "download_sources_report.md"
+    lines = ["# Download sources report", "", "| source | status |", "| --- | --- |"]
+    for k, v in status.items():
+        lines.append(f"| {k} | {'ok' if v else 'missing/failed'} |")
+    path.write_text("\n".join(lines) + "\n", encoding="utf-8")
+    print(f"Wrote download report: {path}")
+
+
+def main() -> int:
+    root = raw_dir()
+    root.mkdir(parents=True, exist_ok=True)
+    status: dict[str, bool] = {}
+
+    status["PolDem"] = download("https://poldem.eui.eu/downloads/cosa/poldem-election_all.csv", root / "poldem" / "poldem-election_all.csv")
+    status["PartyFacts external parties"] = download("https://partyfacts.herokuapp.com/download/external-parties-csv/", root / "partyfacts" / "partyfacts-external-parties.csv")
+    status["Manifesto MPDS 2025a"] = download_manifesto(root / "manifesto" / "MPDataset_MPDS2025a.csv")
+
+    ches_base = "https://www.chesdata.eu/s/"
+    ches_files = {
+        "1999-2019_CHES_dataset_meansv3.csv": "1999-2019_CHES_dataset_means(v3).csv",
+        "1999-2024_CHES_dataset_meansV2-3k4l.csv": "1999-2024_CHES_dataset_meansV2-3k4l.csv",
+        "CHES_2024_final_v2.csv": "CHES_2024_final_v2.csv",
+        "CHES_2024_ALL_Stacked_Expert.csv": "CHES_2024_expert_level.csv",
+        "CHES_CA2023.csv": "CHES_CA2023.csv",
+        "CHES_CA2023_expert-level.csv": "CHES_CA2023_expert_level.csv",
+        "ches_la_2020_aggregate_level_v01.csv": "ches_la_2020_aggregate_level_v01.csv",
+        "ches_la_2020_expert_level_v01.csv": "CHES_LA2020_expert_level.csv",
+        "CHES_ISRAEL_means_2021_2022.csv": "CHES_ISRAEL_means_2021_2022.csv",
+        "CHES_ISRAEL_expert_level_2021_2022.csv": "CHES_IL_expert_level.csv",
+    }
+    for remote, local in ches_files.items():
+        status[f"CHES {local}"] = download(ches_base + urllib.parse.quote(remote), root / "ches" / local)
+
+    status["POPPA integrated v2"] = download_dataverse("doi:10.7910/DVN/RMQREQ", "poppa_integrated_v2.rds", root / "poppa" / "poppa_integrated_v2.rds", "final_data_v2")
+    status["GPS 2019 party"] = download_dataverse("doi:10.7910/DVN/WMGTNS", "Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab", root / "gps" / "Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab")
+    status["V-Party"] = download_vparty(root / "vparty" / "V-Dem-CPD-Party-V2.rds")
+
+    write_report(status)
+    return 0 if all(status.values()) else 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

data-setup/run_data_setup.sh

@@ -0,0 +1,59 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MODE="${1:---dry-run}"
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+cd "$repo_root"
+
+case "$MODE" in
+  --dry-run|--download-only|--build-test|--compare|--full-test) ;;
+  *)
+    echo "Usage: $0 [--dry-run|--download-only|--build-test|--compare|--full-test]" >&2
+    exit 1
+    ;;
+esac
+
+export PARTY2D_RAW_DATA_DIR="${PARTY2D_RAW_DATA_DIR:-$repo_root/_local/raw}"
+export PARTY2D_BUILD_DIR="${PARTY2D_BUILD_DIR:-$repo_root/_local/build}"
+export PARTY2D_GENERATED_INPUT_DIR="${PARTY2D_GENERATED_INPUT_DIR:-$repo_root/_local/generated-inputs}"
+export PARTY2D_REPORT_DIR="${PARTY2D_REPORT_DIR:-$repo_root/_local/reports}"
+export R_LIBS_USER="${R_LIBS_USER:-$repo_root/_local/R/library}"
+mkdir -p "$R_LIBS_USER"
+
+if [ "$MODE" = "--dry-run" ]; then
+  echo "Checking data-setup scripts only; no downloads and no rebuild."
+  command -v bash >/dev/null
+  command -v Rscript >/dev/null
+  command -v python3 >/dev/null
+  bash -n data-setup/check_raw_data.sh
+  python3 -m py_compile data-setup/download_sources.py
+  python3 -m py_compile data-setup/compare_generated_inputs.py
+  Rscript -e 'files <- list.files("data-setup/R", pattern="\\.R$", full.names=TRUE); invisible(lapply(files, parse)); cat("R setup scripts parse OK\n")'
+  echo "Data setup dry run passed."
+  exit 0
+fi
+
+if [ "$MODE" = "--download-only" ]; then
+  python3 data-setup/download_sources.py
+  exit 0
+fi
+
+if [ "$MODE" = "--compare" ]; then
+  python3 data-setup/compare_generated_inputs.py
+  exit 0
+fi
+
+if [ "$MODE" = "--full-test" ]; then
+  python3 data-setup/download_sources.py || true
+fi
+
+bash data-setup/check_raw_data.sh
+rm -rf "$PARTY2D_BUILD_DIR" "$PARTY2D_GENERATED_INPUT_DIR"
+mkdir -p "$PARTY2D_BUILD_DIR" "$PARTY2D_GENERATED_INPUT_DIR"
+Rscript data-setup/R/00_install_dependencies.R
+Rscript data-setup/R/02_build_model_inputs.R
+if [ "$MODE" = "--full-test" ]; then
+  python3 data-setup/compare_generated_inputs.py || true
+else
+  python3 data-setup/compare_generated_inputs.py
+fi

data-setup/source_manifest.csv

@@ -0,0 +1,17 @@
+source,scope,local_path,access,automatic_download,notes
+Manifesto Project,manifesto text/coding,manifesto/MPDataset_MPDS2025a.csv,API key/login required,yes with MANIFESTO_API_KEY,Use the MPDS 2025a CSV export; raw file is not redistributed.
+PolDem Election Campaigns,media campaign issue statements,poldem/poldem-election_all.csv,public download,yes,CSV URL https://poldem.eui.eu/downloads/cosa/poldem-election_all.csv; observed sha256 2cd8c9108b1b0b9c1b6594bb21acee709c70259cd02f450bc69fc09b505fc9fb.
+CHES 1999-2019,expert party placements,ches/1999-2019_CHES_dataset_means(v3).csv,public archived download,yes,Downloaded from archived CHES URL at chesdata.eu.
+CHES 2024,expert party placements,ches/CHES_2024_final_v2.csv,CHES terms,no,Requires matching expert-level file for expert counts.
+CHES 2024 expert level,expert counts,ches/CHES_2024_expert_level.csv,CHES terms,no,Required for expert counts.
+CHES Canada 2023 aggregate,expert party placements,ches/CHES_CA2023.csv,CHES terms,no,Required for Canada extension.
+CHES Canada 2023,expert party placements,ches/CHES_CA2023_expert_level.csv,CHES terms,no,Used by expert-source processing where available.
+CHES Latin America aggregate,expert party placements,ches/ches_la_2020_aggregate_level_v01.csv,CHES terms,no,Required for Latin America extension.
+CHES Latin America 2020,expert party placements,ches/CHES_LA2020_expert_level.csv,CHES terms,no,Used by expert-source processing where available.
+CHES Israel aggregate,expert party placements,ches/CHES_ISRAEL_means_2021_2022.csv,CHES terms,no,Required for Israel extension.
+CHES Israel,expert party placements,ches/CHES_IL_expert_level.csv,CHES terms,no,Used by expert-source processing where available.
+V-Party,expert-coded party variables,vparty/V-Dem-CPD-Party-V2.rds,V-Dem form terms,yes with PARTY2D_VDEM_EMAIL,Downloader submits provider form and extracts R data from ZIP.
+POPPA,expert party placements,poppa/poppa_integrated_v2.rds,public Dataverse,yes,Downloaded from Harvard Dataverse DOI 10.7910/DVN/RMQREQ.
+Global Party Survey 2019,expert party placements,gps/Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab,public Dataverse,yes,Downloaded from Harvard Dataverse DOI 10.7910/DVN/WMGTNS.
+Morgan historical expert data,historical left-right placements,morgan/morgan_positions_raw.csv,derived local transcription/no redistribution,no public URL,Local OCR/transcription source used for historical anchoring.
+PartyFacts crosswalk,party ID harmonization,partyfacts/partyfacts-external-parties.csv,public download,yes,Support crosswalk required by source-processing scripts.

data/README.md

@@ -1,7 +1,13 @@
 # Data directory
 
-This directory contains processed, model-ready party-level inputs used by the estimation pipeline.
+This directory contains only the processed, model-ready inputs used by the Julia/Stan estimation pipeline:
 
-Original raw source files are not stored here. To regenerate the processed inputs, place raw files in a local directory and set `PARTY2D_RAW_DATA_DIR`; see `../docs/RAW_DATA_SOURCES.md`.
+- `text_data.csv`
+- `expert.csv`
+- `lr_data.csv`
+- `union_mapping.csv`
+- `party_families.csv`
+
+Original raw source files and intermediate build products are not stored here. To regenerate the processed inputs, place raw files in a local directory and set `PARTY2D_RAW_DATA_DIR`; see `../data-setup/README.md` and `../docs/RAW_DATA_SOURCES.md`.
 
 Generated outputs and temporary staging files are ignored by git.

docs/RAW_DATA_SOURCES.md

@@ -1,9 +1,13 @@
 # Raw data sources and local-only setup
 
-The public replication repository includes processed inputs under `data/`, but it
+This repository includes model-ready inputs under `data/`, but it does not
-does not redistribute licensed or third-party raw source files. Raw files needed
+redistribute licensed, restricted, or third-party raw source files. Raw files
-to regenerate processed inputs should be kept in a local-only directory outside
+needed to regenerate processed inputs should be kept in `_local/raw/` or another
-the replication git repository.
+local-only directory selected with `PARTY2D_RAW_DATA_DIR`.
+
+The data setup workflow writes intermediates to `_local/build/`, regenerated
+inputs to `_local/generated-inputs/`, and comparison reports to `_local/reports/`.
+It does not overwrite committed `data/` files.
 
 Recommended local layout:
 
@@ -11,6 +15,12 @@ Recommended local layout:
 _local/raw/
   poldem/poldem-election_all.csv
   manifesto/MPDataset_MPDS2025a.csv
+  partyfacts/partyfacts-external-parties.csv
+  ches/...
+  vparty/...
+  poppa/...
+  gps/...
+  morgan/...
 ```
 
 The scripts read `PARTY2D_RAW_DATA_DIR` when it is set. For another location, use:
@@ -23,8 +33,14 @@ export PARTY2D_RAW_DATA_DIR=/path/to/local/raw
 
 | Source | Raw file | Local path below `$PARTY2D_RAW_DATA_DIR` | Used by | Redistribution |
 | --- | --- | --- | --- | --- |
-| PolDem Election Campaigns, all issues | `poldem-election_all.csv` | `poldem/poldem-election_all.csv` | `src/r/00c_process_poldem.R` | Not redistributed in this repo |
+| Manifesto Project Dataset | `MPDataset_MPDS2025a.csv` | `manifesto/MPDataset_MPDS2025a.csv` | `data-setup/R/process_manifesto.R` | Not redistributed in this repo |
-| Manifesto Project Dataset | `MPDataset_MPDS2025a.csv` | `manifesto/MPDataset_MPDS2025a.csv` | `src/r/00a_process_manifesto.R` if `data/manifesto_data.csv` is regenerated | Not redistributed in this repo |
+| PolDem Election Campaigns, all issues | `poldem-election_all.csv` | `poldem/poldem-election_all.csv` | `data-setup/R/process_poldem.R` | Not redistributed in this repo |
+| CHES family | CHES aggregate and expert-level files | `ches/` | `data-setup/R/process_expert.R` | Not redistributed in this repo |
+| V-Party | `V-Dem-CPD-Party-V2.rds` | `vparty/V-Dem-CPD-Party-V2.rds` | `data-setup/R/process_expert.R` | Not redistributed in this repo |
+| POPPA | `poppa_integrated_v2.rds` | `poppa/poppa_integrated_v2.rds` | `data-setup/R/process_expert.R` | Not redistributed in this repo |
+| Global Party Survey 2019 | `Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab` | `gps/Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab` | `data-setup/R/process_expert.R` | Not redistributed in this repo |
+| Morgan historical expert data | `morgan_positions_raw.csv` | `morgan/morgan_positions_raw.csv` | `data-setup/R/process_morgan.R` | Not redistributed in this repo |
+| PartyFacts crosswalk | `partyfacts-external-parties.csv` | `partyfacts/partyfacts-external-parties.csv` | source harmonization scripts | Not redistributed in this repo |
 
 ## PolDem download
 
@@ -46,22 +62,51 @@ sha256  2cd8c9108b1b0b9c1b6594bb21acee709c70259cd02f450bc69fc09b505fc9fb
 From the repository root, run:
 
 ```bash
-bash src/sh/check_raw_data.sh
+bash data-setup/run_data_setup.sh --dry-run
 ```
 
-This checks required local raw inputs and prints byte sizes and SHA-256 checksums.
+This parses the setup scripts without downloading or rebuilding. To check local
+raw file placement and print byte sizes/checksums, run:
+
+```bash
+bash data-setup/check_raw_data.sh
+```
+
+After restricted/manual files are in place, run the full local rebuild and comparison test with:
+
+```bash
+bash data-setup/run_data_setup.sh --full-test
+```
+
+The generated files remain under `_local/generated-inputs/`. They are compared to
+the committed model-ready inputs, but never copied over them automatically.
 
 ## Processed files kept in this repository
 
-The processed files under `data/` are intended to be part of this repository,
+The committed files under `data/` are limited to the model-ready inputs used by
-including:
+the Julia/Stan estimation path:
 
-- `data/poldem_data.csv`
-- `data/manifesto_data.csv`
 - `data/text_data.csv`
 - `data/expert.csv`
 - `data/lr_data.csv`
-- `data/model_data.csv`
+- `data/union_mapping.csv`
+- `data/party_families.csv`
 
 These files document the analysis-ready inputs while avoiding redistribution of
 the underlying raw source data.
+
+See `data-setup/source_manifest.csv` for a machine-readable source checklist.
+
+## Scripted download status
+
+`data-setup/download_sources.py` downloads all sources that are script-accessible
+without project-specific credentials: PolDem, PartyFacts, CHES family files,
+POPPA from Harvard Dataverse, GPS from Harvard Dataverse, and V-Party through the
+provider form when `PARTY2D_VDEM_EMAIL` is set.
+
+The Manifesto Project main dataset requires a Manifesto API key/login. Set
+`MANIFESTO_API_KEY` or `PARTY2D_MANIFESTO_API_KEY` for scripted download.
+
+The Morgan historical file is a local OCR/transcription source from Morgan
+(1976), not a public provider download. It must be placed locally at
+`$PARTY2D_RAW_DATA_DIR/morgan/morgan_positions_raw.csv` for full rebuild tests.

docs/union_mapping.md

@@ -41,7 +41,7 @@ The V4 Stan model (`stan_model_2dim_v4.stan`) uses these mappings to produce **i
 ### Data flow
 
 ```
-R pipeline (00_data-management.R):
+Data setup pipeline (`data-setup/R/02_build_model_inputs.R`):
   - text_data keeps union PF IDs (211) for manifesto rows
   - expert/lr_data keeps individual PF IDs (1375, 1731)
   - Expert filtering: party in text_data OR party in constituent_parties

run_estimation.sh

@@ -17,27 +17,33 @@ export PARTY2D_RAW_DATA_DIR="${PARTY2D_RAW_DATA_DIR:-$repo_root/_local/raw}"
 export TMPDIR="${TMPDIR:-$repo_root/_local/tmp}"
 mkdir -p "$TMPDIR"
 
+required_model_inputs=(
+  "data/text_data.csv"
+  "data/expert.csv"
+  "data/lr_data.csv"
+  "data/union_mapping.csv"
+  "data/party_families.csv"
+)
+
 if [ "$MODE" = "dry-run" ]; then
   echo "Checking commands..."
   command -v bash >/dev/null
-  command -v Rscript >/dev/null
   command -v julia >/dev/null
   echo "Checking key files..."
   test -f Project.toml
   test -f Manifest.toml
   test -f models/stan_model_2dim_v6.stan
-  test -f data/text_data.csv
+  for input in "${required_model_inputs[@]}"; do
-  test -f data/expert.csv
+    test -f "$input"
-  test -f data/lr_data.csv
+  done
-  test -f data/model_data.csv
   echo "Checking shell syntax..."
   bash -n scripts/01_prepare_data.sh
   bash -n scripts/02_fit_model.sh
   bash -n scripts/03_extract_estimates.sh
   bash -n scripts/04_enrich_estimates.sh
   bash -n scripts/05_validate_estimates.sh
-  echo "Checking R syntax..."
+  bash -n data-setup/run_data_setup.sh
-  Rscript -e 'files <- list.files("src/r", pattern="\\.R$", full.names=TRUE); invisible(lapply(files, parse))'
+  bash -n data-setup/check_raw_data.sh
   echo "Checking Julia project can instantiate without running model code..."
   julia --project=. -e 'import Pkg; Pkg.instantiate(); println("Julia project OK")'
   echo "Dry run passed. No model fitting was run."
@@ -54,3 +60,4 @@ fi
 
 bash scripts/03_extract_estimates.sh
 bash scripts/04_enrich_estimates.sh
+bash scripts/05_validate_estimates.sh

scripts/01_prepare_data.sh

@@ -4,4 +4,25 @@ set -euo pipefail
 repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
 cd "$repo_root"
 
-Rscript -e 'setwd("data"); source("../src/r/00_data-management.R")'
+required_model_inputs=(
+  "data/text_data.csv"
+  "data/expert.csv"
+  "data/lr_data.csv"
+  "data/union_mapping.csv"
+  "data/party_families.csv"
+)
+
+missing=0
+for input in "${required_model_inputs[@]}"; do
+  if [ ! -s "$input" ]; then
+    echo "Missing model-ready input: $input" >&2
+    missing=1
+  fi
+done
+
+if [ "$missing" -ne 0 ]; then
+  echo "Regenerate model inputs with data-setup/run_data_setup.sh, or restore the included data/ files." >&2
+  exit 1
+fi
+
+echo "Model-ready data inputs are present."

src/julia/02_enrich_output.jl

@@ -1,20 +1,16 @@
 #!/usr/bin/env julia
 #=
-02_enrich_output.jl - Enrich party positions CSV with metadata
+02_enrich_output.jl - Enrich party positions CSV with model-input-derived metadata
 
-Fast post-processing script that adds party names, union membership status,
+Fast post-processing script that adds union membership status from the model-ready
-and election results to the model output CSV. Operates purely on CSV files
+inputs. Operates purely on CSV files (no chain loading). Takes seconds, not minutes.
-(no chain loading). Takes seconds, not minutes.
 
 Usage:
     julia 02_enrich_output.jl                # enriches latest party_positions_*.csv
     julia 02_enrich_output.jl somefile.csv    # enriches a specific file
 
 Adds columns:
-    party_name_english  - Full English party name (from data/party_names.csv)
-    party_name_short    - Standard abbreviation
     in_union            - 1 if party's union had a joint manifesto that year, 0 otherwise
-    pervote             - Vote share (%) at election years; missing for non-election years
 =#
 
 using CSV
@@ -40,32 +36,7 @@ function enrich(input_file::String)
     output = CSV.read(input_file, DataFrame)
     println("  Rows: $(nrow(output)), Columns: $(ncol(output))")
 
-    # --- Party names ---
+    # --- Union mapping (for in_union) ---
-    party_names_file = joinpath("data", "party_names.csv")
-    if isfile(party_names_file)
-        names_df = CSV.read(party_names_file, DataFrame)
-        name_lookup = Dict{Int, Tuple{String, String}}()
-        for row in eachrow(names_df)
-            short = ismissing(row.party_name_short) ? "" : string(row.party_name_short)
-            name_lookup[row.partyfacts_id] = (string(row.party_name_english), short)
-        end
-        output.party_name_english = [
-            haskey(name_lookup, pid) ? name_lookup[pid][1] : ""
-            for pid in output.party_id
-        ]
-        output.party_name_short = [
-            haskey(name_lookup, pid) ? name_lookup[pid][2] : ""
-            for pid in output.party_id
-        ]
-        n_named = count(x -> x != "", output.party_name_english)
-        println("  Party names: $n_named / $(nrow(output)) rows matched")
-    else
-        output.party_name_english = fill("", nrow(output))
-        output.party_name_short = fill("", nrow(output))
-        println("  WARNING: data/party_names.csv not found")
-    end
-
-    # --- Union mapping (for in_union + pervote fallback) ---
     union_mapping_file = joinpath("data", "union_mapping.csv")
     constituent_to_union = Dict{Int, Int}()
     if isfile(union_mapping_file)
@@ -126,49 +97,6 @@ function enrich(input_file::String)
         println("  WARNING: Could not compute in_union (missing files)")
     end
 
-    # --- Election results (pervote) ---
-    election_file = joinpath("data", "election_data.csv")
-    output.pervote = Vector{Union{Float64, Missing}}(missing, nrow(output))
-
-    if isfile(election_file)
-        election_df = CSV.read(election_file, DataFrame)
-
-        # Build lookup: (party_id, year) -> pervote
-        election_lookup = Dict{Tuple{Int, Int}, Float64}()
-        for row in eachrow(election_df)
-            election_lookup[(row.party, row.year)] = row.pervote
-        end
-
-        n_filled = 0
-        for i in 1:nrow(output)
-            pid = output.party_id[i]
-            yr = output.year[i]
-
-            # Direct match (standalone party)
-            if haskey(election_lookup, (pid, yr))
-                output.pervote[i] = election_lookup[(pid, yr)]
-                n_filled += 1
-            elseif hasproperty(output, :union_party_id) && !ismissing(output.union_party_id[i])
-                # Union constituent: look up by union PF ID
-                uid = output.union_party_id[i]
-                if haskey(election_lookup, (uid, yr))
-                    output.pervote[i] = election_lookup[(uid, yr)]
-                    n_filled += 1
-                end
-            elseif haskey(constituent_to_union, pid)
-                # Fallback: use union mapping even if union_party_id column not populated
-                uid = constituent_to_union[pid]
-                if haskey(election_lookup, (uid, yr))
-                    output.pervote[i] = election_lookup[(uid, yr)]
-                    n_filled += 1
-                end
-            end
-        end
-        println("  pervote: $n_filled / $(nrow(output)) rows filled")
-    else
-        println("  WARNING: election_data.csv not found")
-    end
-
     # --- Reorder columns ---
     estimate_cols = Symbol[]
     for base in ["economic_lr", "galtan", "pro_market", "pro_welfare", "cosmopolitan", "traditional"]
@@ -185,8 +113,7 @@ function enrich(input_file::String)
     output.country = replace(output.country, "MO" => "MK")
 
     col_order = vcat(
-        [:party_id, :party_name_english, :party_name_short, :country, :year, :segment_num,
+        [:party_id, :country, :year, :segment_num, :union_party_id, :in_union],
-         :union_party_id, :in_union, :pervote],
         estimate_cols
     )
     col_order = filter(c -> hasproperty(output, c), col_order)