party2d/src/r/00c_process_poldem.R

# ============================================================
# 00c_process_poldem.R - PolDem Media Data Processing
# ============================================================
# Processes PolDem (Political Deliberation in the Media) data
# for the 4D latent trait model
#
# Input: $PARTY2D_RAW_DATA_DIR/poldem/poldem-election_all.csv (sentence-level)
# Output: poldem_data.csv (party-year-var aggregates)
# ============================================================

library(tidyverse)
library(countrycode)

# Set working directory (works both in RStudio and command line)
if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
}

cat("Processing PolDem media data...\n")

raw_data_dir <- Sys.getenv(
  "PARTY2D_RAW_DATA_DIR",
  unset = file.path("..", "..", "_local", "raw")
)
poldem_raw_path <- file.path(raw_data_dir, "poldem", "poldem-election_all.csv")

# ============================================================
# PartyFacts Linkage (via CMP party IDs)
# ============================================================

partyfacts_raw <- read_csv('partyfacts-external-parties.csv', show_col_types = FALSE)
manifesto_link <- partyfacts_raw %>%
  filter(dataset_key == "manifesto") %>%
  transmute(cmp = as.numeric(dataset_party_id),  # Convert to numeric for join
            country_pf = countrycode(country, origin = 'iso3c', destination = "iso2c"),
            party = partyfacts_id,
            party = ifelse(party == 622, 604, party))

# ============================================================
# Issue Category Mapping to 4 Dimensions
# ============================================================
# For positive direction: type_high is the active trait
# For negative direction: we flip (same data, just contributes to the opposite trait)

poldem_mapping <- tribble(
  ~issue_cat,     ~dimension,  ~type_high,      ~type_low,
  # Economic dimension
  "ecolib",       "economic",  "pro_market",    "pro_welfare",   # Economic liberalization
  "welfare",      "economic",  "pro_welfare",   "pro_market",    # Welfare state
  # Final exclusion: the PolDem economic-reform category is intentionally
  # omitted because item-response diagnostics showed that it did not load
  # substantively onto the economic latent trait.
  # Cultural dimension
  "immig",        "cultural",  "cosmopolitan",  "traditional",   # Immigration (pro = cosmopolitan)
  "cultlib",      "cultural",  "cosmopolitan",  "traditional",   # Cultural liberalism
  "nationalism",  "cultural",  "traditional",   "cosmopolitan",  # Nationalism (pro = traditional)
  "europe",       "cultural",  "cosmopolitan",  "traditional",   # EU integration (pro = cosmopolitan)
  "euro",         "cultural",  "cosmopolitan",  "traditional",   # Euro currency (pro = cosmopolitan)
  "defense",      "cultural",  "traditional",   "cosmopolitan",  # Defense (pro = traditional)
  "security",     "cultural",  "traditional",   "cosmopolitan"   # Security/law-order (pro = traditional)
)

cat(sprintf("  Using %d issue categories\n", nrow(poldem_mapping)))

# ============================================================
# Load and Clean PolDem Data
# ============================================================

poldem_raw <- read_csv(poldem_raw_path, show_col_types = FALSE)
cat(sprintf("  Raw PolDem data: %d rows\n", nrow(poldem_raw)))

poldem <- poldem_raw %>%
  # Fix country codes
  mutate(country = case_when(
    iso2code == "AU" ~ "AT",  # Austria (PolDem uses AU instead of AT)
    iso2code == "UK" ~ "GB",  # United Kingdom
    TRUE ~ iso2code
  )) %>%
  # Extract year from article date (format: YYYY-MM-DD)
  mutate(year = suppressWarnings(as.numeric(substr(date_art, 1, 4)))) %>%
  # Filter to valid issue categories only
  filter(issue_cat %in% poldem_mapping$issue_cat) %>%
  # Convert direction to numeric and filter out neutral/NA
  mutate(direction = as.numeric(direction)) %>%
  filter(!is.na(direction) & direction != 0) %>%
  # Remove rows with invalid years
  filter(!is.na(year))

cat(sprintf("  After filtering: %d rows (valid issues, non-neutral)\n", nrow(poldem)))

# ============================================================
# Link to PartyFacts via CMP codes
# ============================================================

poldem <- poldem %>%
  mutate(cmp = as.numeric(cmp)) %>%
  left_join(manifesto_link, by = "cmp") %>%
  filter(!is.na(party))

# Report linkage
n_linked <- nrow(poldem)
n_unlinked <- nrow(poldem_raw %>%
                     filter(issue_cat %in% poldem_mapping$issue_cat) %>%
                     mutate(direction = as.numeric(direction)) %>%
                     filter(!is.na(direction) & direction != 0)) - n_linked

cat(sprintf("  Linked to PartyFacts: %d rows\n", n_linked))
if (n_unlinked > 0) {
  cat(sprintf("  Warning: %d rows could not be linked (missing CMP mapping)\n", n_unlinked))
}

# ============================================================
# Aggregate to Party-Year-Issue Level
# Using round(sum()) for weak direction values (0.5, -0.5)
# ============================================================

poldem_agg <- poldem %>%
  left_join(poldem_mapping, by = "issue_cat") %>%
  group_by(party, country, year, issue_cat, type_high, type_low) %>%
  summarise(
    # Sum positive directions (0.5 and 1), then round
    positive = round(sum(direction[direction > 0])),
    # Sum absolute directions for sample (all non-neutral), then round
    sample = round(sum(abs(direction))),
    n_obs = n(),
    .groups = "drop"
  ) %>%
  # Minimum 3 observations per group
  filter(n_obs >= 3) %>%
  select(-n_obs)

cat(sprintf("  After aggregation: %d party-year-issue observations\n", nrow(poldem_agg)))

# ============================================================
# Format Output (matching manifesto structure)
# ============================================================

poldem_data <- poldem_agg %>%
  mutate(
    var = paste0(issue_cat, "_poldem"),
    project = "PolDem"
  ) %>%
  select(party, country, year, var, positive, sample, type_high, type_low, project)

# ============================================================
# Write Output
# ============================================================

write_csv(poldem_data, "poldem_data.csv")

cat(sprintf("\nOutput: poldem_data.csv\n"))
cat(sprintf("  Total rows: %d\n", nrow(poldem_data)))
cat(sprintf("  Unique parties: %d\n", n_distinct(poldem_data$party)))
cat(sprintf("  Countries: %s\n", paste(sort(unique(poldem_data$country)), collapse = ", ")))
cat(sprintf("  Year range: %d-%d\n", min(poldem_data$year, na.rm = TRUE), max(poldem_data$year, na.rm = TRUE)))
cat("\n  Rows by issue category:\n")
poldem_data %>%
  group_by(var) %>%
  summarise(n = n(), .groups = "drop") %>%
  arrange(desc(n)) %>%
  print()