Wire local-only data setup workflow
This commit is contained in:
aseimel
@@ -0,0 +1,179 @@
|
||||
# ============================================================
|
||||
# process_manifesto.R - Manifesto Project Data Processing
|
||||
# ============================================================
|
||||
# Processes Manifesto Project data for the 4D latent trait model
|
||||
# Input: $PARTY2D_RAW_DATA_DIR/manifesto/MPDataset_MPDS2025a.csv
|
||||
# Output: manifesto_data.csv
|
||||
# ============================================================
|
||||
|
||||
library(tidyverse)
|
||||
library(countrycode)
|
||||
library(purrr)
|
||||
|
||||
# Set working directory (works both in RStudio and command line)
|
||||
if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
|
||||
try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
|
||||
}
|
||||
|
||||
cat("Processing Manifesto Project data...\n")
|
||||
|
||||
raw_data_dir <- Sys.getenv(
|
||||
"PARTY2D_RAW_DATA_DIR",
|
||||
unset = file.path("..", "..", "_local", "raw")
|
||||
)
|
||||
manifesto_raw_path <- file.path(raw_data_dir, "manifesto", "MPDataset_MPDS2025a.csv")
|
||||
partyfacts_path <- file.path(raw_data_dir, "partyfacts", "partyfacts-external-parties.csv")
|
||||
|
||||
# ============================================================
|
||||
# PartyFacts Linkage
|
||||
# ============================================================
|
||||
|
||||
partyfacts_raw <- read_csv(partyfacts_path, show_col_types = FALSE)
|
||||
manifesto_link <- partyfacts_raw %>%
|
||||
filter(dataset_key == "manifesto") %>%
|
||||
transmute(id = dataset_party_id,
|
||||
country = countrycode(country, origin = 'iso3c', destination = "iso2c"),
|
||||
party = partyfacts_id,
|
||||
party = ifelse(party == 622, 604, party))
|
||||
|
||||
# ============================================================
|
||||
# Load Manifesto Data
|
||||
# ============================================================
|
||||
|
||||
manifesto_data <- read_csv(manifesto_raw_path, show_col_types = FALSE)
|
||||
|
||||
# ============================================================
|
||||
# CMP Code Mapping to 4 Dimensions
|
||||
# ============================================================
|
||||
|
||||
vars <- tribble(
|
||||
~type, ~subtype, ~per_var, ~stance, ~label,
|
||||
# pro_market
|
||||
"pro_market", "Market Regulation", "per401", "Positive", "Free Market Economy",
|
||||
"pro_market", "Economic Liberalization","per402", "Positive", "Incentives: Positive",
|
||||
"pro_market", "Market Regulation", "per407", "Positive", "Protectionism: Negative",
|
||||
"pro_market", "Economic Liberalization","per414", "Positive", "Economic Orthodoxy",
|
||||
"pro_market", "Economic Liberalization","per505", "Positive", "Welfare State Limitation",
|
||||
"pro_market", "Economic Liberalization","per507", "Positive", "Education Limitation",
|
||||
"pro_market", "Economic Liberalization","per702", "Positive", "Labour Groups: Negative",
|
||||
"pro_market", "Market Regulation", "per406", "Negative", "Protectionism: Positive",
|
||||
"pro_market", "Market Regulation", "per412", "Negative", "Controlled Economy",
|
||||
"pro_market", "Economic Liberalization","per504", "Negative", "Welfare State Expansion",
|
||||
# pro_welfare
|
||||
"pro_welfare", "Economic Intervention", "per403", "Positive", "Market Regulation",
|
||||
"pro_welfare", "Economic Intervention", "per404", "Positive", "Economic Planning",
|
||||
"pro_welfare", "Economic Intervention", "per412", "Positive", "Controlled Economy",
|
||||
"pro_welfare", "Economic Intervention", "per413", "Positive", "Nationalisation",
|
||||
"pro_welfare", "Social Services", "per504", "Positive", "Welfare State Expansion",
|
||||
"pro_welfare", "Social Services", "per506", "Positive", "Education Expansion",
|
||||
"pro_welfare", "Economic Intervention", "per701", "Positive", "Labour Groups: Positive",
|
||||
"pro_welfare", "Economic Intervention", "per401", "Negative", "Free Market Economy",
|
||||
"pro_welfare", "Social Services", "per505", "Negative", "Welfare State Limitation",
|
||||
# cosmopolitan
|
||||
"cosmopolitan", "Internationalism", "per107", "Positive", "Internationalism: Positive",
|
||||
"cosmopolitan", "Internationalism", "per108", "Positive", "European Community/Union: Positive",
|
||||
"cosmopolitan", "Multiculturalism", "per607", "Positive", "Multiculturalism: Positive",
|
||||
"cosmopolitan", "Multiculturalism", "per201", "Positive", "Freedom and Human Rights",
|
||||
"cosmopolitan", "Multiculturalism", "per604", "Positive", "traditional Morality: Negative",
|
||||
"cosmopolitan", "Internationalism", "per109", "Negative", "Internationalism: Negative",
|
||||
"cosmopolitan", "Multiculturalism", "per601", "Negative", "National Way of Life: Positive",
|
||||
# traditional
|
||||
"traditional", "National Identity", "per109", "Positive", "Internationalism: Negative",
|
||||
"traditional", "Conservative Morality", "per110", "Positive", "European Community/Union: Negative",
|
||||
"traditional", "National Identity", "per601", "Positive", "National Way of Life: Positive",
|
||||
"traditional", "Conservative Morality", "per603", "Positive", "traditional Morality: Positive",
|
||||
"traditional", "Conservative Morality", "per608", "Positive", "Multiculturalism: Negative",
|
||||
"traditional", "Conservative Morality", "per605", "Positive", "Law and Order: Positive",
|
||||
"traditional", "National Identity", "per107", "Negative", "Internationalism: Positive",
|
||||
"traditional", "Conservative Morality", "per607", "Negative", "Multiculturalism: Positive"
|
||||
)
|
||||
|
||||
# ============================================================
|
||||
# Process Manifesto Data
|
||||
# ============================================================
|
||||
|
||||
manifesto <- vars %>%
|
||||
pmap_dfr(~ manifesto_data %>%
|
||||
transmute(country = countrycode(countryname, origin = 'country.name', destination = 'iso2c'),
|
||||
year = as.numeric(format(as.Date(edate, format = "%d/%m/%Y"), "%Y")),
|
||||
id = as.character(party),
|
||||
count = round(.data[[..3]]),
|
||||
var = ..3,
|
||||
label = ..5,
|
||||
type = ..1,
|
||||
subtype = ..2,
|
||||
stance = ..4,
|
||||
project = 'Manifesto Project') %>%
|
||||
left_join(manifesto_link, by = c("id", "country")) %>%
|
||||
select(-id)) %>%
|
||||
group_by(party, country, year, subtype) %>%
|
||||
summarise(
|
||||
positive = sum(count[stance == "Positive"], na.rm = TRUE),
|
||||
sample = sum(count, na.rm = TRUE),
|
||||
type = first(type),
|
||||
project = first(project),
|
||||
.groups = "drop"
|
||||
) %>%
|
||||
na.omit() %>%
|
||||
rename(var = subtype) %>%
|
||||
# Convert to bipolar bridge structure (type_high/type_low)
|
||||
mutate(
|
||||
type_high = case_when(
|
||||
type == "pro_welfare" ~ "pro_welfare",
|
||||
type == "pro_market" ~ "pro_market",
|
||||
type == "cosmopolitan" ~ "cosmopolitan",
|
||||
type == "traditional" ~ "traditional"
|
||||
),
|
||||
type_low = case_when(
|
||||
type %in% c("pro_welfare", "pro_market") ~ ifelse(type == "pro_welfare", "pro_market", "pro_welfare"),
|
||||
type %in% c("cosmopolitan", "traditional") ~ ifelse(type == "cosmopolitan", "traditional", "cosmopolitan")
|
||||
)
|
||||
) %>%
|
||||
select(-type) %>%
|
||||
# Add _manifesto suffix to variable names
|
||||
mutate(var = paste0(tolower(gsub(" ", "_", var)), "_manifesto"))
|
||||
|
||||
# ============================================================
|
||||
# NOTE: Temporal continuity filter moved to the data setup orchestrator.
|
||||
# This allows exempting parties that appear in parliamentary data
|
||||
# (parties in parliament are by definition not fringe parties)
|
||||
# ============================================================
|
||||
|
||||
cat("Skipping temporal filter (applied in 02_build_model_inputs.R after combining with other text data)\n")
|
||||
cat(sprintf(" Parties: %d\n", n_distinct(manifesto$party)))
|
||||
|
||||
# ============================================================
|
||||
# Write Output
|
||||
# ============================================================
|
||||
|
||||
write_csv(manifesto, "manifesto_data.csv")
|
||||
cat(sprintf("Output: manifesto_data.csv (%d rows, %d parties)\n",
|
||||
nrow(manifesto), n_distinct(manifesto$party)))
|
||||
|
||||
# ============================================================
|
||||
# Election Data Extraction (vote shares)
|
||||
# ============================================================
|
||||
|
||||
cat("\nExtracting election data (pervote)...\n")
|
||||
|
||||
election_data <- manifesto_data %>%
|
||||
transmute(
|
||||
country = countrycode(countryname, origin = 'country.name', destination = 'iso2c'),
|
||||
year = as.numeric(format(as.Date(edate, format = "%d/%m/%Y"), "%Y")),
|
||||
id = as.character(party),
|
||||
pervote = pervote
|
||||
) %>%
|
||||
left_join(manifesto_link, by = c("id", "country")) %>%
|
||||
select(-id) %>%
|
||||
filter(!is.na(party), !is.na(pervote)) %>%
|
||||
# Keep one row per (party, country, year) — take max pervote if duplicates
|
||||
group_by(party, country, year) %>%
|
||||
summarise(pervote = max(pervote, na.rm = TRUE), .groups = "drop") %>%
|
||||
arrange(country, party, year)
|
||||
|
||||
write_csv(election_data, "election_data.csv")
|
||||
cat(sprintf("Output: election_data.csv (%d rows, %d parties)\n",
|
||||
nrow(election_data), n_distinct(election_data$party)))
|
||||
|
||||
# Export manifesto_link for use by other scripts
|
||||
# (poldem also needs it for CMP linkage)
|
||||
Reference in New Issue
Block a user