Document raw source file reference
This commit is contained in:
aseimel
@@ -0,0 +1,161 @@
|
||||
# ============================================================
|
||||
# 00c_process_poldem.R - PolDem Media Data Processing
|
||||
# ============================================================
|
||||
# Processes PolDem (Political Deliberation in the Media) data
|
||||
# for the 4D latent trait model
|
||||
#
|
||||
# Input: $PARTY2D_RAW_DATA_DIR/poldem/poldem-election_all.csv (sentence-level)
|
||||
# Output: poldem_data.csv (party-year-var aggregates)
|
||||
# ============================================================
|
||||
|
||||
library(tidyverse)
|
||||
library(countrycode)
|
||||
|
||||
# Set working directory (works both in RStudio and command line)
|
||||
if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
|
||||
try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
|
||||
}
|
||||
|
||||
cat("Processing PolDem media data...\n")
|
||||
|
||||
raw_data_dir <- Sys.getenv(
|
||||
"PARTY2D_RAW_DATA_DIR",
|
||||
unset = file.path("..", "..", "_local", "raw")
|
||||
)
|
||||
poldem_raw_path <- file.path(raw_data_dir, "poldem", "poldem-election_all.csv")
|
||||
|
||||
# ============================================================
|
||||
# PartyFacts Linkage (via CMP party IDs)
|
||||
# ============================================================
|
||||
|
||||
partyfacts_raw <- read_csv('partyfacts-external-parties.csv', show_col_types = FALSE)
|
||||
manifesto_link <- partyfacts_raw %>%
|
||||
filter(dataset_key == "manifesto") %>%
|
||||
transmute(cmp = as.numeric(dataset_party_id), # Convert to numeric for join
|
||||
country_pf = countrycode(country, origin = 'iso3c', destination = "iso2c"),
|
||||
party = partyfacts_id,
|
||||
party = ifelse(party == 622, 604, party))
|
||||
|
||||
# ============================================================
|
||||
# Issue Category Mapping to 4 Dimensions
|
||||
# ============================================================
|
||||
# For positive direction: type_high is the active trait
|
||||
# For negative direction: we flip (same data, just contributes to the opposite trait)
|
||||
|
||||
poldem_mapping <- tribble(
|
||||
~issue_cat, ~dimension, ~type_high, ~type_low,
|
||||
# Economic dimension
|
||||
"ecolib", "economic", "pro_market", "pro_welfare", # Economic liberalization
|
||||
"welfare", "economic", "pro_welfare", "pro_market", # Welfare state
|
||||
# Final exclusion: the PolDem economic-reform category is intentionally
|
||||
# omitted because item-response diagnostics showed that it did not load
|
||||
# substantively onto the economic latent trait.
|
||||
# Cultural dimension
|
||||
"immig", "cultural", "cosmopolitan", "traditional", # Immigration (pro = cosmopolitan)
|
||||
"cultlib", "cultural", "cosmopolitan", "traditional", # Cultural liberalism
|
||||
"nationalism", "cultural", "traditional", "cosmopolitan", # Nationalism (pro = traditional)
|
||||
"europe", "cultural", "cosmopolitan", "traditional", # EU integration (pro = cosmopolitan)
|
||||
"euro", "cultural", "cosmopolitan", "traditional", # Euro currency (pro = cosmopolitan)
|
||||
"defense", "cultural", "traditional", "cosmopolitan", # Defense (pro = traditional)
|
||||
"security", "cultural", "traditional", "cosmopolitan" # Security/law-order (pro = traditional)
|
||||
)
|
||||
|
||||
cat(sprintf(" Using %d issue categories\n", nrow(poldem_mapping)))
|
||||
|
||||
# ============================================================
|
||||
# Load and Clean PolDem Data
|
||||
# ============================================================
|
||||
|
||||
poldem_raw <- read_csv(poldem_raw_path, show_col_types = FALSE)
|
||||
cat(sprintf(" Raw PolDem data: %d rows\n", nrow(poldem_raw)))
|
||||
|
||||
poldem <- poldem_raw %>%
|
||||
# Fix country codes
|
||||
mutate(country = case_when(
|
||||
iso2code == "AU" ~ "AT", # Austria (PolDem uses AU instead of AT)
|
||||
iso2code == "UK" ~ "GB", # United Kingdom
|
||||
TRUE ~ iso2code
|
||||
)) %>%
|
||||
# Extract year from article date (format: YYYY-MM-DD)
|
||||
mutate(year = suppressWarnings(as.numeric(substr(date_art, 1, 4)))) %>%
|
||||
# Filter to valid issue categories only
|
||||
filter(issue_cat %in% poldem_mapping$issue_cat) %>%
|
||||
# Convert direction to numeric and filter out neutral/NA
|
||||
mutate(direction = as.numeric(direction)) %>%
|
||||
filter(!is.na(direction) & direction != 0) %>%
|
||||
# Remove rows with invalid years
|
||||
filter(!is.na(year))
|
||||
|
||||
cat(sprintf(" After filtering: %d rows (valid issues, non-neutral)\n", nrow(poldem)))
|
||||
|
||||
# ============================================================
|
||||
# Link to PartyFacts via CMP codes
|
||||
# ============================================================
|
||||
|
||||
poldem <- poldem %>%
|
||||
mutate(cmp = as.numeric(cmp)) %>%
|
||||
left_join(manifesto_link, by = "cmp") %>%
|
||||
filter(!is.na(party))
|
||||
|
||||
# Report linkage
|
||||
n_linked <- nrow(poldem)
|
||||
n_unlinked <- nrow(poldem_raw %>%
|
||||
filter(issue_cat %in% poldem_mapping$issue_cat) %>%
|
||||
mutate(direction = as.numeric(direction)) %>%
|
||||
filter(!is.na(direction) & direction != 0)) - n_linked
|
||||
|
||||
cat(sprintf(" Linked to PartyFacts: %d rows\n", n_linked))
|
||||
if (n_unlinked > 0) {
|
||||
cat(sprintf(" Warning: %d rows could not be linked (missing CMP mapping)\n", n_unlinked))
|
||||
}
|
||||
|
||||
# ============================================================
|
||||
# Aggregate to Party-Year-Issue Level
|
||||
# Using round(sum()) for weak direction values (0.5, -0.5)
|
||||
# ============================================================
|
||||
|
||||
poldem_agg <- poldem %>%
|
||||
left_join(poldem_mapping, by = "issue_cat") %>%
|
||||
group_by(party, country, year, issue_cat, type_high, type_low) %>%
|
||||
summarise(
|
||||
# Sum positive directions (0.5 and 1), then round
|
||||
positive = round(sum(direction[direction > 0])),
|
||||
# Sum absolute directions for sample (all non-neutral), then round
|
||||
sample = round(sum(abs(direction))),
|
||||
n_obs = n(),
|
||||
.groups = "drop"
|
||||
) %>%
|
||||
# Minimum 3 observations per group
|
||||
filter(n_obs >= 3) %>%
|
||||
select(-n_obs)
|
||||
|
||||
cat(sprintf(" After aggregation: %d party-year-issue observations\n", nrow(poldem_agg)))
|
||||
|
||||
# ============================================================
|
||||
# Format Output (matching manifesto structure)
|
||||
# ============================================================
|
||||
|
||||
poldem_data <- poldem_agg %>%
|
||||
mutate(
|
||||
var = paste0(issue_cat, "_poldem"),
|
||||
project = "PolDem"
|
||||
) %>%
|
||||
select(party, country, year, var, positive, sample, type_high, type_low, project)
|
||||
|
||||
# ============================================================
|
||||
# Write Output
|
||||
# ============================================================
|
||||
|
||||
write_csv(poldem_data, "poldem_data.csv")
|
||||
|
||||
cat(sprintf("\nOutput: poldem_data.csv\n"))
|
||||
cat(sprintf(" Total rows: %d\n", nrow(poldem_data)))
|
||||
cat(sprintf(" Unique parties: %d\n", n_distinct(poldem_data$party)))
|
||||
cat(sprintf(" Countries: %s\n", paste(sort(unique(poldem_data$country)), collapse = ", ")))
|
||||
cat(sprintf(" Year range: %d-%d\n", min(poldem_data$year, na.rm = TRUE), max(poldem_data$year, na.rm = TRUE)))
|
||||
cat("\n Rows by issue category:\n")
|
||||
poldem_data %>%
|
||||
group_by(var) %>%
|
||||
summarise(n = n(), .groups = "drop") %>%
|
||||
arrange(desc(n)) %>%
|
||||
print()
|
||||
Reference in New Issue
Block a user