Publish party-position estimates and validation materials
This commit is contained in:
Armin Seimel
@@ -0,0 +1,986 @@
|
||||
#!/usr/bin/env julia
|
||||
#=
|
||||
02_post_estimation.jl - Extract party position estimates from Stan model output
|
||||
|
||||
Supports both model versions:
|
||||
- 2D model (V1): Extracts economic left-right and cultural cosmopolitan--traditionalist positions directly
|
||||
- 4D model (V10): Extracts 4 traits + 2 derived scales
|
||||
|
||||
V10/V1 UPDATE: Handles segment-based indexing
|
||||
- Maps segment results back to original party IDs
|
||||
- Adds segment_num column to indicate which segment of the party
|
||||
- Flags discontinuities for parties with multiple segments
|
||||
|
||||
This script:
|
||||
1. Auto-detects the latest model run in model_outputs/
|
||||
2. Loads the chain CSV files and data mappings
|
||||
3. Detects model version from metadata or column names
|
||||
4. Extracts posterior summaries for all segment-year positions
|
||||
5. Maps Stan parameter indices back to real party IDs, segment numbers, and years
|
||||
6. Saves output as wide-format CSV with uncertainty estimates
|
||||
|
||||
Usage:
|
||||
julia 02_post_estimation.jl
|
||||
|
||||
Output:
|
||||
party_positions_YYYY-MM-DD_HH-MM-SS.csv
|
||||
=#
|
||||
|
||||
using CSV
|
||||
using DataFrames
|
||||
using Statistics
|
||||
using JSON
|
||||
using Dates
|
||||
using Printf
|
||||
|
||||
# =============================================================================
|
||||
# STEP 0: Auto-detect latest run
|
||||
# =============================================================================
|
||||
|
||||
function find_latest_run(base_dir::String="outputs/model_outputs/latest")
|
||||
if !isdir(base_dir)
|
||||
error("Model outputs directory not found: $base_dir")
|
||||
end
|
||||
|
||||
runs = filter(d -> startswith(d, "run_") && isdir(joinpath(base_dir, d)), readdir(base_dir))
|
||||
|
||||
if isempty(runs)
|
||||
error("No runs found in $base_dir")
|
||||
end
|
||||
|
||||
# Sort by timestamp in directory name (format: run_YYYY-MM-DD_HH-MM-SS)
|
||||
sort!(runs, rev=true)
|
||||
|
||||
latest = joinpath(base_dir, runs[1])
|
||||
println("Found $(length(runs)) run(s). Using latest: $latest")
|
||||
return latest
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 1: Load data and build segment-year lookup
|
||||
# =============================================================================
|
||||
|
||||
function load_run_data(run_dir::String)
|
||||
println("\n" * "="^60)
|
||||
println("LOADING RUN DATA")
|
||||
println("="^60)
|
||||
|
||||
data_dir = joinpath(run_dir, "data")
|
||||
chains_dir = joinpath(run_dir, "chains")
|
||||
|
||||
# Check required files exist
|
||||
required_files = [
|
||||
joinpath(data_dir, "text_data.csv"),
|
||||
joinpath(data_dir, "expert_dim.csv"),
|
||||
joinpath(data_dir, "expert_lr.csv"),
|
||||
joinpath(run_dir, "metadata.json")
|
||||
]
|
||||
|
||||
for f in required_files
|
||||
if !isfile(f)
|
||||
error("Required file not found: $f")
|
||||
end
|
||||
end
|
||||
|
||||
# Load data files
|
||||
println("Loading text_data.csv...")
|
||||
text_data = CSV.read(joinpath(data_dir, "text_data.csv"), DataFrame)
|
||||
println(" Rows: $(nrow(text_data))")
|
||||
|
||||
println("Loading expert_dim.csv...")
|
||||
expert_dim = CSV.read(joinpath(data_dir, "expert_dim.csv"), DataFrame)
|
||||
println(" Rows: $(nrow(expert_dim))")
|
||||
|
||||
println("Loading expert_lr.csv...")
|
||||
expert_lr = CSV.read(joinpath(data_dir, "expert_lr.csv"), DataFrame)
|
||||
println(" Rows: $(nrow(expert_lr))")
|
||||
|
||||
println("Loading metadata.json...")
|
||||
metadata = JSON.parsefile(joinpath(run_dir, "metadata.json"))
|
||||
println(" year0: $(metadata["year0"])")
|
||||
println(" Model: $(metadata["model_file"])")
|
||||
|
||||
# V10: Load segment_info if available
|
||||
segment_info_file = joinpath(data_dir, "segment_info.csv")
|
||||
segment_info = nothing
|
||||
if isfile(segment_info_file)
|
||||
println("Loading segment_info.csv (V10)...")
|
||||
segment_info = CSV.read(segment_info_file, DataFrame)
|
||||
println(" Segments: $(nrow(segment_info))")
|
||||
end
|
||||
|
||||
# V10: Load segment_year_map if available
|
||||
segment_year_file = joinpath(data_dir, "segment_year_map.csv")
|
||||
segment_year_map = nothing
|
||||
if isfile(segment_year_file)
|
||||
println("Loading segment_year_map.csv (V10)...")
|
||||
segment_year_map = CSV.read(segment_year_file, DataFrame)
|
||||
println(" Segment-years: $(nrow(segment_year_map))")
|
||||
end
|
||||
|
||||
# Find chain files
|
||||
chain_files = filter(f -> endswith(f, ".csv") && startswith(f, "chain_"), readdir(chains_dir))
|
||||
println("\nFound $(length(chain_files)) chain file(s)")
|
||||
|
||||
return (
|
||||
text_data = text_data,
|
||||
expert_dim = expert_dim,
|
||||
expert_lr = expert_lr,
|
||||
metadata = metadata,
|
||||
segment_info = segment_info,
|
||||
segment_year_map = segment_year_map,
|
||||
chain_files = [joinpath(chains_dir, f) for f in sort(chain_files)],
|
||||
run_dir = run_dir
|
||||
)
|
||||
end
|
||||
|
||||
function normalize_country_value(value)
|
||||
if ismissing(value)
|
||||
return missing
|
||||
end
|
||||
txt = strip(string(value))
|
||||
return isempty(txt) ? missing : txt
|
||||
end
|
||||
|
||||
function build_party_country_map(text_data::DataFrame, expert_dim::DataFrame, expert_lr::DataFrame)
|
||||
merged = unique(vcat(
|
||||
select(text_data, :party, :country),
|
||||
select(expert_dim, :party, :country),
|
||||
select(expert_lr, :party, :country)
|
||||
))
|
||||
|
||||
party_to_country = Dict{Int, String}()
|
||||
for row in eachrow(merged)
|
||||
pid = tryparse(Int, string(row.party))
|
||||
if pid === nothing
|
||||
continue
|
||||
end
|
||||
c = normalize_country_value(row.country)
|
||||
if !ismissing(c)
|
||||
party_to_country[pid] = c
|
||||
end
|
||||
end
|
||||
return party_to_country
|
||||
end
|
||||
|
||||
function load_constituent_to_union_map()::Dict{Int, Int}
|
||||
mapping_file = joinpath("data", "union_mapping.csv")
|
||||
constituent_to_union = Dict{Int, Int}()
|
||||
if isfile(mapping_file)
|
||||
union_df = CSV.read(mapping_file, DataFrame)
|
||||
for row in eachrow(union_df)
|
||||
constituent_to_union[row.expert_pf_id] = row.manifesto_pf_id
|
||||
end
|
||||
end
|
||||
return constituent_to_union
|
||||
end
|
||||
|
||||
function resolve_party_country(pid_value,
|
||||
party_to_country::Dict{Int, String},
|
||||
constituent_to_union::Dict{Int, Int})
|
||||
pid = tryparse(Int, string(pid_value))
|
||||
if pid === nothing
|
||||
return missing, "unresolved"
|
||||
end
|
||||
|
||||
if haskey(party_to_country, pid)
|
||||
return party_to_country[pid], "direct"
|
||||
end
|
||||
|
||||
if haskey(constituent_to_union, pid)
|
||||
uid = constituent_to_union[pid]
|
||||
if haskey(party_to_country, uid)
|
||||
return party_to_country[uid], "union_fallback"
|
||||
end
|
||||
end
|
||||
|
||||
return missing, "unresolved"
|
||||
end
|
||||
|
||||
function apply_country_resolution!(df::DataFrame,
|
||||
party_col::Symbol,
|
||||
country_col::Symbol,
|
||||
party_to_country::Dict{Int, String},
|
||||
constituent_to_union::Dict{Int, Int})
|
||||
resolved_country = Union{Missing, String}[]
|
||||
source_counts = Dict("direct" => 0, "union_fallback" => 0, "unresolved" => 0)
|
||||
unresolved_parties = Set{Int}()
|
||||
|
||||
for pid in df[!, party_col]
|
||||
country, source = resolve_party_country(pid, party_to_country, constituent_to_union)
|
||||
push!(resolved_country, country)
|
||||
source_counts[source] += 1
|
||||
if source == "unresolved"
|
||||
pid_int = tryparse(Int, string(pid))
|
||||
if pid_int !== nothing
|
||||
push!(unresolved_parties, pid_int)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
df[!, country_col] = resolved_country
|
||||
return source_counts, sort!(collect(unresolved_parties))
|
||||
end
|
||||
|
||||
function fill_missing_countries!(df::DataFrame,
|
||||
segment_info::Union{DataFrame, Nothing},
|
||||
party_to_country::Dict{Int, String},
|
||||
constituent_to_union::Dict{Int, Int})
|
||||
if !hasproperty(df, :country)
|
||||
source_counts, unresolved = apply_country_resolution!(
|
||||
df, :party_id, :country, party_to_country, constituent_to_union
|
||||
)
|
||||
return source_counts, unresolved
|
||||
end
|
||||
|
||||
normalized = Union{Missing, String}[]
|
||||
for val in df.country
|
||||
push!(normalized, normalize_country_value(val))
|
||||
end
|
||||
df.country = normalized
|
||||
|
||||
source_counts = Dict("direct" => 0, "union_fallback" => 0, "segment_info" => 0, "unresolved" => 0)
|
||||
|
||||
segment_country_by_id = Dict{Int, String}()
|
||||
if segment_info !== nothing && hasproperty(segment_info, :country)
|
||||
for row in eachrow(segment_info)
|
||||
c = normalize_country_value(row.country)
|
||||
if !ismissing(c)
|
||||
segment_country_by_id[Int(row.segment_id)] = c
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
unresolved_parties = Set{Int}()
|
||||
for i in 1:nrow(df)
|
||||
if !ismissing(df.country[i])
|
||||
continue
|
||||
end
|
||||
|
||||
if hasproperty(df, :segment_id) && haskey(segment_country_by_id, Int(df.segment_id[i]))
|
||||
df.country[i] = segment_country_by_id[Int(df.segment_id[i])]
|
||||
source_counts["segment_info"] += 1
|
||||
continue
|
||||
end
|
||||
|
||||
country, source = resolve_party_country(df.party_id[i], party_to_country, constituent_to_union)
|
||||
if !ismissing(country)
|
||||
df.country[i] = country
|
||||
source_counts[source] += 1
|
||||
else
|
||||
source_counts["unresolved"] += 1
|
||||
pid_int = tryparse(Int, string(df.party_id[i]))
|
||||
pid_int !== nothing && push!(unresolved_parties, pid_int)
|
||||
end
|
||||
end
|
||||
|
||||
return source_counts, sort!(collect(unresolved_parties))
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 2: Build complete segment-year mapping (V10) or party-year mapping (V9)
|
||||
# =============================================================================
|
||||
|
||||
function build_segment_year_map(text_data::DataFrame, expert_dim::DataFrame, expert_lr::DataFrame,
|
||||
segment_info::Union{DataFrame, Nothing},
|
||||
segment_year_map::Union{DataFrame, Nothing},
|
||||
run_dir::String,
|
||||
year0::Int)
|
||||
println("\n" * "="^60)
|
||||
println("BUILDING SEGMENT-YEAR MAPPING")
|
||||
println("="^60)
|
||||
|
||||
party_to_country = build_party_country_map(text_data, expert_dim, expert_lr)
|
||||
constituent_to_union = load_constituent_to_union_map()
|
||||
|
||||
# V10: Use segment_year_map if available
|
||||
if segment_year_map !== nothing && segment_info !== nothing
|
||||
println("Using segment_year_map.csv (V10 mode)")
|
||||
|
||||
# Convert relative Year to absolute year
|
||||
if hasproperty(segment_year_map, :Year)
|
||||
segment_year_map.year = segment_year_map.Year .+ year0
|
||||
elseif !hasproperty(segment_year_map, :year)
|
||||
error("segment_year_map has no Year or year column")
|
||||
end
|
||||
|
||||
# Add party_id from segment_info if not already present
|
||||
if !hasproperty(segment_year_map, :party_id)
|
||||
segment_id_to_party = Dict(row.segment_id => row.party_id for row in eachrow(segment_info))
|
||||
segment_year_map.party_id = [segment_id_to_party[sid] for sid in segment_year_map.segment_id]
|
||||
end
|
||||
|
||||
# Add segment_num from segment_info if not already present
|
||||
if !hasproperty(segment_year_map, :segment_num)
|
||||
segment_id_to_segnum = Dict(row.segment_id => row.segment_num for row in eachrow(segment_info))
|
||||
segment_year_map.segment_num = [segment_id_to_segnum[sid] for sid in segment_year_map.segment_id]
|
||||
end
|
||||
|
||||
# Resolve/fill country column using segment metadata first, then direct and union-fallback lookup.
|
||||
source_counts, unresolved = fill_missing_countries!(
|
||||
segment_year_map, segment_info, party_to_country, constituent_to_union
|
||||
)
|
||||
direct_count = get(source_counts, "direct", 0)
|
||||
union_count = get(source_counts, "union_fallback", 0)
|
||||
segment_info_count = get(source_counts, "segment_info", 0)
|
||||
unresolved_count = count(ismissing, segment_year_map.country)
|
||||
println(" Country resolution fill counts: direct=$direct_count, union_fallback=$union_count, segment_info=$segment_info_count, unresolved_rows=$unresolved_count")
|
||||
if !isempty(unresolved)
|
||||
println(" Warning: unresolved country party IDs (first 20): $(unresolved[1:min(20, length(unresolved))])")
|
||||
end
|
||||
|
||||
R = maximum(segment_year_map.rr)
|
||||
n_segments = length(unique(segment_year_map.segment_id))
|
||||
n_parties = length(unique(segment_year_map.party_id))
|
||||
|
||||
println("Loaded segment_year_map: $(nrow(segment_year_map)) segment-years (R=$R)")
|
||||
println(" Unique segments: $n_segments")
|
||||
println(" Unique parties: $n_parties")
|
||||
|
||||
# Count observed vs interpolated
|
||||
observed_rrs = Set{Int}()
|
||||
if hasproperty(text_data, :rr_man)
|
||||
union!(observed_rrs, Set(text_data.rr_man))
|
||||
end
|
||||
if hasproperty(expert_dim, :rr_exp_dim)
|
||||
union!(observed_rrs, Set(expert_dim.rr_exp_dim))
|
||||
end
|
||||
if hasproperty(expert_lr, :rr_exp_lr)
|
||||
union!(observed_rrs, Set(expert_lr.rr_exp_lr))
|
||||
end
|
||||
|
||||
n_observed = length(intersect(Set(segment_year_map.rr), observed_rrs))
|
||||
n_interpolated = nrow(segment_year_map) - n_observed
|
||||
println(" Observed segment-years: $n_observed")
|
||||
println(" Interpolated segment-years: $n_interpolated")
|
||||
|
||||
return segment_year_map, R, segment_info
|
||||
end
|
||||
|
||||
# V9 fallback: Use party_year_map
|
||||
party_year_file = joinpath(run_dir, "data", "party_year_map.csv")
|
||||
if isfile(party_year_file)
|
||||
println("Loading party_year_map.csv (V9 fallback mode)")
|
||||
party_year_map = CSV.read(party_year_file, DataFrame)
|
||||
|
||||
# Add party_id column (same as party for V9)
|
||||
if !hasproperty(party_year_map, :party_id)
|
||||
party_year_map.party_id = party_year_map.party
|
||||
end
|
||||
|
||||
# Add segment_num column (always 1 for V9)
|
||||
if !hasproperty(party_year_map, :segment_num)
|
||||
party_year_map.segment_num = ones(Int, nrow(party_year_map))
|
||||
end
|
||||
|
||||
# Add segment_id column (same as party index for V9)
|
||||
if !hasproperty(party_year_map, :segment_id)
|
||||
party_year_map.segment_id = party_year_map.party
|
||||
end
|
||||
|
||||
# Convert relative Year to absolute year
|
||||
if hasproperty(party_year_map, :Year)
|
||||
party_year_map.year = party_year_map.Year .+ year0
|
||||
elseif !hasproperty(party_year_map, :year)
|
||||
error("party_year_map has no Year or year column")
|
||||
end
|
||||
|
||||
# Resolve/fill country column
|
||||
if !hasproperty(party_year_map, :country)
|
||||
source_counts, unresolved = apply_country_resolution!(
|
||||
party_year_map, :party_id, :country, party_to_country, constituent_to_union
|
||||
)
|
||||
direct_count = source_counts["direct"]
|
||||
union_count = source_counts["union_fallback"]
|
||||
unresolved_count = source_counts["unresolved"]
|
||||
println(" Country resolution sources: direct=$direct_count, union_fallback=$union_count, unresolved=$unresolved_count")
|
||||
if !isempty(unresolved)
|
||||
println(" Warning: unresolved country party IDs (first 20): $(unresolved[1:min(20, length(unresolved))])")
|
||||
end
|
||||
else
|
||||
normalized = Union{Missing, String}[]
|
||||
for val in party_year_map.country
|
||||
push!(normalized, normalize_country_value(val))
|
||||
end
|
||||
party_year_map.country = normalized
|
||||
end
|
||||
|
||||
R = maximum(party_year_map.rr)
|
||||
println("Loaded party_year_map: $(nrow(party_year_map)) party-years (R=$R)")
|
||||
|
||||
return party_year_map, R, nothing
|
||||
end
|
||||
|
||||
# Fallback: Reconstruct from data files
|
||||
@warn "No mapping file found, reconstructing from data (observed years only)"
|
||||
|
||||
# Extract unique party-year-rr combinations from text_data
|
||||
text_map = unique(select(text_data, :party, :country, :year, :rr_man))
|
||||
rename!(text_map, :rr_man => :rr)
|
||||
text_map.party_id = text_map.party
|
||||
text_map.segment_num = ones(Int, nrow(text_map))
|
||||
|
||||
expert_dim_map = unique(select(expert_dim, :party, :country, :year, :rr_exp_dim))
|
||||
rename!(expert_dim_map, :rr_exp_dim => :rr)
|
||||
expert_dim_map.party_id = expert_dim_map.party
|
||||
expert_dim_map.segment_num = ones(Int, nrow(expert_dim_map))
|
||||
|
||||
expert_lr_map = unique(select(expert_lr, :party, :country, :year, :rr_exp_lr))
|
||||
rename!(expert_lr_map, :rr_exp_lr => :rr)
|
||||
expert_lr_map.party_id = expert_lr_map.party
|
||||
expert_lr_map.segment_num = ones(Int, nrow(expert_lr_map))
|
||||
|
||||
combined = vcat(text_map, expert_dim_map, expert_lr_map)
|
||||
segment_year_map = unique(combined)
|
||||
sort!(segment_year_map, :rr)
|
||||
|
||||
R = maximum(segment_year_map.rr)
|
||||
println("Reconstructed mapping: $(nrow(segment_year_map)) segment-years (R=$R)")
|
||||
|
||||
return segment_year_map, R, nothing
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 3: Load and combine chains
|
||||
# =============================================================================
|
||||
|
||||
function load_chains(chain_files::Vector{String})
|
||||
println("\n" * "="^60)
|
||||
println("LOADING STAN CHAINS")
|
||||
println("="^60)
|
||||
flush(stdout)
|
||||
|
||||
chains = DataFrame[]
|
||||
|
||||
# The full Stan CSVs are very wide (hundreds of thousands of columns). For
|
||||
# post-estimation we only need party-position generated quantities. Reading
|
||||
# all columns can take hours and allocate many GB of irrelevant parameters.
|
||||
post_estimation_prefixes = (
|
||||
"economic_lr.",
|
||||
"galtan.",
|
||||
"pro_market.",
|
||||
"pro_welfare.",
|
||||
"cosmopolitan.",
|
||||
"traditional.",
|
||||
)
|
||||
keep_post_estimation_col(_i, name) = any(startswith(String(name), p) for p in post_estimation_prefixes)
|
||||
|
||||
for (i, f) in enumerate(chain_files)
|
||||
println("Loading chain $i: $(basename(f))...")
|
||||
flush(stdout)
|
||||
# Skip comment lines (Stan header) and parse only needed quantities.
|
||||
chain = CSV.read(f, DataFrame; comment="#", select=keep_post_estimation_col)
|
||||
println(" Samples: $(nrow(chain)), Parameters: $(ncol(chain))")
|
||||
flush(stdout)
|
||||
push!(chains, chain)
|
||||
end
|
||||
|
||||
# Combine chains
|
||||
println("Combining selected chain columns...")
|
||||
flush(stdout)
|
||||
combined = vcat(chains...)
|
||||
println("\nCombined: $(nrow(combined)) total samples")
|
||||
println("Selected parameters: $(ncol(combined))")
|
||||
flush(stdout)
|
||||
|
||||
return combined
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 4: Extract generated quantities
|
||||
# =============================================================================
|
||||
|
||||
"""
|
||||
Detect model version from chain column names.
|
||||
Returns "2dim" or "4dim".
|
||||
"""
|
||||
function detect_model_version(chains::DataFrame)
|
||||
cols = names(chains)
|
||||
# 2D model has economic_lr but NOT pro_market
|
||||
has_economic_lr = any(c -> startswith(string(c), "economic_lr."), cols)
|
||||
has_pro_market = any(c -> startswith(string(c), "pro_market."), cols)
|
||||
|
||||
if has_economic_lr && !has_pro_market
|
||||
return "2dim"
|
||||
elseif has_pro_market
|
||||
return "4dim"
|
||||
else
|
||||
error("Could not detect model version from chain columns")
|
||||
end
|
||||
end
|
||||
|
||||
function extract_estimates(chains::DataFrame, segment_year_map::DataFrame, R::Int)
|
||||
println("\n" * "="^60)
|
||||
println("EXTRACTING POSTERIOR ESTIMATES")
|
||||
println("="^60)
|
||||
|
||||
# Auto-detect model version from columns
|
||||
model_version = detect_model_version(chains)
|
||||
println("Detected model version: $model_version")
|
||||
|
||||
# Select quantities based on model version
|
||||
if model_version == "2dim"
|
||||
# 2D model: economic left-right and cultural cosmopolitan--traditionalist positions are directly estimated
|
||||
# (general_lr is computed in Stan for anchoring but not extracted as output)
|
||||
quantities = ["economic_lr", "galtan"]
|
||||
test_col = "economic_lr.1"
|
||||
else
|
||||
# 4D model: 4 traits + 2 derived scales
|
||||
quantities = ["pro_market", "pro_welfare", "cosmopolitan", "traditional", "economic_lr", "galtan"]
|
||||
test_col = "pro_market.1"
|
||||
end
|
||||
|
||||
# Check that columns exist
|
||||
if !hasproperty(chains, Symbol(test_col))
|
||||
error("Column $test_col not found in chains. Available columns: $(first(names(chains), 10))...")
|
||||
end
|
||||
|
||||
n_samples = nrow(chains)
|
||||
println("Samples per parameter: $n_samples")
|
||||
|
||||
# Load union mapping for adding union_party_id column
|
||||
union_mapping_file = joinpath("data", "union_mapping.csv")
|
||||
constituent_to_union_pf = Dict{Int, Int}()
|
||||
if isfile(union_mapping_file)
|
||||
union_df = CSV.read(union_mapping_file, DataFrame)
|
||||
for row in eachrow(union_df)
|
||||
constituent_to_union_pf[row.expert_pf_id] = row.manifesto_pf_id
|
||||
end
|
||||
end
|
||||
|
||||
# Pre-allocate output DataFrame
|
||||
n_rows = nrow(segment_year_map)
|
||||
|
||||
# Add union_party_id column: NA for standalone parties, union PF ID for constituents
|
||||
union_ids = Union{Int, Missing}[]
|
||||
for pid in segment_year_map.party_id
|
||||
pid_int = isa(pid, Integer) ? pid : tryparse(Int, string(pid))
|
||||
if pid_int !== nothing && haskey(constituent_to_union_pf, pid_int)
|
||||
push!(union_ids, constituent_to_union_pf[pid_int])
|
||||
else
|
||||
push!(union_ids, missing)
|
||||
end
|
||||
end
|
||||
|
||||
output = DataFrame(
|
||||
party_id = segment_year_map.party_id,
|
||||
union_party_id = union_ids,
|
||||
segment_num = segment_year_map.segment_num,
|
||||
country = segment_year_map.country,
|
||||
year = segment_year_map.year,
|
||||
rr = segment_year_map.rr
|
||||
)
|
||||
|
||||
# Add columns for each quantity
|
||||
for q in quantities
|
||||
output[!, Symbol(q)] = zeros(Float64, n_rows)
|
||||
output[!, Symbol("$(q)_se")] = zeros(Float64, n_rows)
|
||||
output[!, Symbol("$(q)_q025")] = zeros(Float64, n_rows)
|
||||
output[!, Symbol("$(q)_q975")] = zeros(Float64, n_rows)
|
||||
end
|
||||
|
||||
println("Extracting estimates for $(n_rows) segment-year positions...")
|
||||
|
||||
# Progress tracking
|
||||
prog_interval = max(1, n_rows ÷ 20)
|
||||
|
||||
for (i, row) in enumerate(eachrow(segment_year_map))
|
||||
r = row.rr
|
||||
|
||||
# Progress
|
||||
if i % prog_interval == 0 || i == n_rows
|
||||
pct = round(100 * i / n_rows, digits=1)
|
||||
print("\r Progress: $pct% ($i / $n_rows)")
|
||||
end
|
||||
|
||||
for q in quantities
|
||||
col_name = Symbol("$q.$r")
|
||||
|
||||
if !hasproperty(chains, col_name)
|
||||
@warn "Column $col_name not found (rr=$r)" maxlog=5
|
||||
continue
|
||||
end
|
||||
|
||||
samples = chains[!, col_name]
|
||||
|
||||
# Compute summary statistics
|
||||
output[i, Symbol(q)] = mean(samples)
|
||||
output[i, Symbol("$(q)_se")] = std(samples)
|
||||
output[i, Symbol("$(q)_q025")] = quantile(samples, 0.025)
|
||||
output[i, Symbol("$(q)_q975")] = quantile(samples, 0.975)
|
||||
end
|
||||
end
|
||||
println() # Newline after progress
|
||||
|
||||
# Remove the rr column from final output (internal only)
|
||||
select!(output, Not(:rr))
|
||||
|
||||
return output
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 5: Validation
|
||||
# =============================================================================
|
||||
|
||||
function validate_output(output::DataFrame, segment_info::Union{DataFrame, Nothing})
|
||||
println("\n" * "="^60)
|
||||
println("VALIDATION CHECKS")
|
||||
println("="^60)
|
||||
|
||||
all_passed = true
|
||||
|
||||
# Detect which columns are present (2D vs 4D model)
|
||||
has_4d = hasproperty(output, :pro_market)
|
||||
|
||||
# Check 1: Range check - all estimates should be in [0, 1]
|
||||
println("\n1. Range check (all values in [0, 1]):")
|
||||
|
||||
if has_4d
|
||||
check_cols = [:pro_market, :pro_welfare, :cosmopolitan, :traditional, :economic_lr, :galtan]
|
||||
else
|
||||
check_cols = [:economic_lr, :galtan]
|
||||
end
|
||||
|
||||
for col in check_cols
|
||||
if !hasproperty(output, col)
|
||||
continue
|
||||
end
|
||||
vals = output[!, col]
|
||||
min_val, max_val = extrema(vals)
|
||||
in_range = min_val >= 0 && max_val <= 1
|
||||
status = in_range ? "PASS" : "FAIL"
|
||||
println(" $col: [$(@sprintf("%.4f", min_val)), $(@sprintf("%.4f", max_val))] - $status")
|
||||
all_passed = all_passed && in_range
|
||||
end
|
||||
|
||||
# Check 2: Anchor party checks
|
||||
println("\n2. Anchor party checks:")
|
||||
|
||||
# Define anchor parties with expected ranges (for 2D model)
|
||||
# Includes both union IDs (V3) and individual constituent IDs (V4)
|
||||
anchor_parties = [
|
||||
(id=211, name="CDU/CSU", country="DE", econ=(0.50, 0.70), galtan=(0.45, 0.70)),
|
||||
(id=1375, name="CDU", country="DE", econ=(0.50, 0.70), galtan=(0.45, 0.65)),
|
||||
(id=1731, name="CSU", country="DE", econ=(0.50, 0.70), galtan=(0.55, 0.75)),
|
||||
(id=383, name="SPD", country="DE", econ=(0.30, 0.50), galtan=(0.30, 0.55)),
|
||||
(id=1516, name="Labour", country="GB", econ=(0.30, 0.55), galtan=(0.30, 0.55)),
|
||||
(id=1567, name="Conservatives", country="GB", econ=(0.55, 0.80), galtan=(0.50, 0.75)),
|
||||
(id=487, name="SAP", country="SE", econ=(0.30, 0.50), galtan=(0.35, 0.55)),
|
||||
]
|
||||
|
||||
n_checked = 0
|
||||
n_passed = 0
|
||||
|
||||
for anchor in anchor_parties
|
||||
party_rows = filter(r -> r.party_id == anchor.id, output)
|
||||
|
||||
if nrow(party_rows) == 0
|
||||
println(" $(anchor.name) ($(anchor.id)): NOT FOUND")
|
||||
continue
|
||||
end
|
||||
|
||||
# Use most recent 20 years of data as reference period
|
||||
max_year = maximum(party_rows.year)
|
||||
ref_rows = filter(r -> r.year >= max_year - 20, party_rows)
|
||||
if nrow(ref_rows) == 0
|
||||
ref_rows = party_rows
|
||||
end
|
||||
|
||||
n_checked += 1
|
||||
|
||||
mean_econ = mean(ref_rows.economic_lr)
|
||||
mean_galtan = mean(ref_rows.galtan)
|
||||
|
||||
econ_ok = anchor.econ[1] <= mean_econ <= anchor.econ[2]
|
||||
galtan_ok = anchor.galtan[1] <= mean_galtan <= anchor.galtan[2]
|
||||
all_ok = econ_ok && galtan_ok
|
||||
|
||||
if all_ok
|
||||
n_passed += 1
|
||||
end
|
||||
|
||||
status = all_ok ? "PASS" : "WARN"
|
||||
econ_marker = econ_ok ? "" : "*"
|
||||
galtan_marker = galtan_ok ? "" : "*"
|
||||
|
||||
@printf(" %-15s economic=%.2f%s [%.2f-%.2f] cultural=%.2f%s [%.2f-%.2f] %s\n",
|
||||
anchor.name, mean_econ, econ_marker, anchor.econ[1], anchor.econ[2],
|
||||
mean_galtan, galtan_marker, anchor.galtan[1], anchor.galtan[2], status)
|
||||
end
|
||||
|
||||
if n_checked > 0
|
||||
println(" Anchor check: $n_passed/$n_checked within expected ranges")
|
||||
println(" Note: Model integrates text + expert data; deviations from expert-only expectations are normal")
|
||||
end
|
||||
|
||||
# Check 3: Coverage check
|
||||
println("\n3. Coverage check:")
|
||||
println(" Total segment-year positions: $(nrow(output))")
|
||||
println(" Unique parties: $(length(unique(output.party_id)))")
|
||||
blank_country_rows = count(ismissing, output.country)
|
||||
println(" Unique countries: $(length(unique(skipmissing(output.country))))")
|
||||
if blank_country_rows == 0
|
||||
println(" Blank country rows: 0 - PASS")
|
||||
else
|
||||
println(" Blank country rows: $blank_country_rows - FAIL")
|
||||
all_passed = false
|
||||
end
|
||||
println(" Year range: $(minimum(output.year)) - $(maximum(output.year))")
|
||||
|
||||
# V10: Check segment distribution
|
||||
segment_counts = combine(groupby(output, :party_id), nrow => :n_years,
|
||||
:segment_num => (x -> length(unique(x))) => :n_segments)
|
||||
parties_multi_segment = filter(:n_segments => >(1), segment_counts)
|
||||
if nrow(parties_multi_segment) > 0
|
||||
println("\n Parties with multiple segments: $(length(unique(parties_multi_segment.party_id)))")
|
||||
end
|
||||
|
||||
# Check 4: No duplicates (party_id, segment_num, year should be unique)
|
||||
println("\n4. Duplicate check:")
|
||||
dup_count = nrow(output) - nrow(unique(select(output, :party_id, :segment_num, :year)))
|
||||
if dup_count == 0
|
||||
println(" No duplicate (party_id, segment_num, year) combinations - PASS")
|
||||
else
|
||||
println(" WARNING: Found $dup_count duplicate combinations!")
|
||||
all_passed = false
|
||||
end
|
||||
|
||||
# Check 5: SE reasonableness
|
||||
println("\n5. Standard error check:")
|
||||
se_cols = has_4d ?
|
||||
[:pro_market_se, :pro_welfare_se, :cosmopolitan_se, :traditional_se] :
|
||||
[:economic_lr_se, :galtan_se]
|
||||
|
||||
for col in se_cols
|
||||
if !hasproperty(output, col)
|
||||
continue
|
||||
end
|
||||
vals = output[!, col]
|
||||
mean_se = mean(vals)
|
||||
max_se = maximum(vals)
|
||||
println(" $col: mean=$(@sprintf("%.4f", mean_se)), max=$(@sprintf("%.4f", max_se))")
|
||||
end
|
||||
|
||||
println("\n" * "-"^60)
|
||||
if all_passed
|
||||
println("All validation checks PASSED")
|
||||
else
|
||||
println("Some validation checks FAILED - please inspect output carefully")
|
||||
end
|
||||
|
||||
return all_passed
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 6: Save output
|
||||
# =============================================================================
|
||||
|
||||
function save_output(output::DataFrame, metadata::Dict, segment_info::Union{DataFrame, Nothing}, run_dir::String; outdir::String="outputs/estimations/latest")
|
||||
println("\n" * "="^60)
|
||||
println("SAVING OUTPUT")
|
||||
println("="^60)
|
||||
|
||||
timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")
|
||||
mkpath(outdir)
|
||||
|
||||
# Delete previous output files
|
||||
for f in readdir(outdir)
|
||||
if startswith(f, "party_positions_") && (endswith(f, ".csv") || endswith(f, ".txt") || endswith(f, ".tex"))
|
||||
rm(joinpath(outdir, f))
|
||||
println(" Deleted old: $f")
|
||||
end
|
||||
end
|
||||
|
||||
# Save main CSV
|
||||
csv_file = joinpath(outdir, "party_positions_$timestamp.csv")
|
||||
CSV.write(csv_file, output)
|
||||
println("Saved: $csv_file")
|
||||
println(" Rows: $(nrow(output))")
|
||||
println(" Columns: $(ncol(output))")
|
||||
|
||||
# Count parties with multiple segments
|
||||
n_multi_segment = 0
|
||||
if segment_info !== nothing
|
||||
party_segment_counts = combine(groupby(segment_info, :party_id), nrow => :n_segments)
|
||||
n_multi_segment = count(r -> r.n_segments > 1, eachrow(party_segment_counts))
|
||||
end
|
||||
|
||||
# Save metadata
|
||||
meta_file = joinpath(outdir, "party_positions_$(timestamp)_metadata.txt")
|
||||
open(meta_file, "w") do f
|
||||
println(f, "Party Positions Dataset - Metadata")
|
||||
println(f, "="^50)
|
||||
println(f, "")
|
||||
println(f, "Generated: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))")
|
||||
println(f, "Source run: $(basename(run_dir))")
|
||||
println(f, "Model file: $(get(metadata, "model_file", "unknown"))")
|
||||
println(f, "")
|
||||
println(f, "Dataset size:")
|
||||
println(f, " Segment-year observations: $(nrow(output))")
|
||||
println(f, " Unique parties: $(length(unique(output.party_id)))")
|
||||
if n_multi_segment > 0
|
||||
println(f, " Parties with multiple segments: $n_multi_segment")
|
||||
end
|
||||
println(f, " Unique countries: $(length(unique(output.country)))")
|
||||
println(f, " Year range: $(minimum(output.year)) - $(maximum(output.year))")
|
||||
println(f, "")
|
||||
println(f, "Columns:")
|
||||
println(f, " party_id: PartyFacts ID (integer) - individual party (e.g., CDU=1375, CSU=1731)")
|
||||
println(f, " union_party_id: PartyFacts ID of parent union (NA for standalone parties)")
|
||||
println(f, " segment_num: Segment number within party (1, 2, 3...)")
|
||||
println(f, " country: ISO2 country code")
|
||||
println(f, " year: Calendar year")
|
||||
println(f, "")
|
||||
println(f, "Segment-Based Indexing:")
|
||||
println(f, " - Parties are split into segments at gaps > 7 years")
|
||||
println(f, " - Each segment is estimated independently (no continuity across gaps)")
|
||||
println(f, " - Segments with < 3 observations are dropped")
|
||||
println(f, " - segment_num=1 is the main segment; higher numbers indicate gaps in data")
|
||||
println(f, "")
|
||||
|
||||
# Check if this is 2D or 4D output
|
||||
is_2d = !hasproperty(output, :pro_market)
|
||||
|
||||
if is_2d
|
||||
println(f, "Model: 2D Direct Bipolar")
|
||||
println(f, "")
|
||||
println(f, "Bipolar scales (0 = left/cosmopolitan, 1 = right/traditionalist):")
|
||||
println(f, " economic_lr: Economic left-right position (directly estimated)")
|
||||
println(f, " galtan: Cultural cosmopolitan--traditionalist position (directly estimated)")
|
||||
# Note: general_lr is computed internally for cross-dimensional anchoring
|
||||
# but not reported as output (the two dimension-specific estimates are preferred)
|
||||
else
|
||||
println(f, "Model: 4D Unipolar")
|
||||
println(f, "")
|
||||
println(f, "Dimensions (0 = low, 1 = high):")
|
||||
println(f, " pro_market: Pro-market economic position")
|
||||
println(f, " pro_welfare: Pro-welfare state position")
|
||||
println(f, " cosmopolitan: Cosmopolitan cultural position")
|
||||
println(f, " traditional: Traditionalist cultural position")
|
||||
println(f, "")
|
||||
println(f, "Derived bipolar scales (0 = left/cosmopolitan, 1 = right/traditionalist):")
|
||||
println(f, " economic_lr: Economic left-right (derived from pro_market - pro_welfare)")
|
||||
println(f, " galtan: Cultural cosmopolitan--traditionalist (derived from traditional - cosmopolitan)")
|
||||
end
|
||||
println(f, "")
|
||||
println(f, "Uncertainty columns:")
|
||||
println(f, " *_se: Standard error (posterior SD)")
|
||||
println(f, " *_q025: 2.5th percentile (lower 95% CI)")
|
||||
println(f, " *_q975: 97.5th percentile (upper 95% CI)")
|
||||
println(f, "")
|
||||
println(f, "Model convergence:")
|
||||
println(f, " Mean R-hat: $(get(metadata, "mean_rhat", "N/A"))")
|
||||
println(f, " Max R-hat: $(get(metadata, "max_rhat", "N/A"))")
|
||||
println(f, " Mean ESS: $(get(metadata, "mean_ess", "N/A"))")
|
||||
println(f, " Min ESS: $(get(metadata, "min_ess", "N/A"))")
|
||||
end
|
||||
println("Saved: $meta_file")
|
||||
|
||||
return csv_file, meta_file
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 5b: Verify no union/alliance IDs in output
|
||||
# =============================================================================
|
||||
|
||||
function verify_no_unions_in_output(output::DataFrame)
|
||||
println("\n" * "="^60)
|
||||
println("UNION ID VERIFICATION")
|
||||
println("="^60)
|
||||
|
||||
union_mapping_file = joinpath("data", "union_mapping.csv")
|
||||
if !isfile(union_mapping_file)
|
||||
println(" No union_mapping.csv found — skipping verification")
|
||||
return
|
||||
end
|
||||
|
||||
union_df = CSV.read(union_mapping_file, DataFrame)
|
||||
union_pf_ids = Set(union_df.manifesto_pf_id)
|
||||
output_pf_ids = Set(output.party_id)
|
||||
|
||||
violations = intersect(union_pf_ids, output_pf_ids)
|
||||
|
||||
if isempty(violations)
|
||||
println(" PASS: No union/alliance PF IDs found in output")
|
||||
println(" Checked $(length(union_pf_ids)) union IDs against $(length(output_pf_ids)) output parties")
|
||||
else
|
||||
println(" WARNING: $(length(violations)) union PF IDs found in output")
|
||||
println(" (This is expected if union_mapping.csv was updated after the model run)")
|
||||
for v in sort(collect(violations))
|
||||
n_rows = count(r -> r.party_id == v, eachrow(output))
|
||||
println(" PF $v: $n_rows rows")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# MAIN
|
||||
# =============================================================================
|
||||
|
||||
function main()
|
||||
println("="^60)
|
||||
println("POST-ESTIMATION: Party-position model")
|
||||
println("="^60)
|
||||
println("Started: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))")
|
||||
|
||||
# Step 0: Find run directory (CLI --run-dir or auto-detect latest)
|
||||
run_dir = nothing
|
||||
output_dir = nothing
|
||||
for (i, arg) in enumerate(ARGS)
|
||||
if arg == "--run-dir" && i < length(ARGS)
|
||||
run_dir = ARGS[i + 1]
|
||||
elseif startswith(arg, "--run-dir=")
|
||||
run_dir = split(arg, "=", limit=2)[2]
|
||||
elseif arg == "--output-dir" && i < length(ARGS)
|
||||
output_dir = ARGS[i + 1]
|
||||
elseif startswith(arg, "--output-dir=")
|
||||
output_dir = split(arg, "=", limit=2)[2]
|
||||
end
|
||||
end
|
||||
if run_dir === nothing
|
||||
run_dir = find_latest_run()
|
||||
else
|
||||
println("Using specified run directory: $run_dir")
|
||||
end
|
||||
|
||||
# Step 1: Load run data
|
||||
data = load_run_data(run_dir)
|
||||
|
||||
# Step 2: Build segment-year mapping
|
||||
year0 = data.metadata["year0"]
|
||||
segment_year_map, R, segment_info = build_segment_year_map(
|
||||
data.text_data, data.expert_dim, data.expert_lr,
|
||||
data.segment_info, data.segment_year_map, data.run_dir, year0
|
||||
)
|
||||
|
||||
# Step 3: Load chains
|
||||
chains = load_chains(data.chain_files)
|
||||
|
||||
# Step 4: Extract estimates
|
||||
output = extract_estimates(chains, segment_year_map, R)
|
||||
|
||||
# Step 5: Validate
|
||||
validate_output(output, segment_info)
|
||||
|
||||
# Step 5b: Verify no union/alliance IDs in output
|
||||
verify_no_unions_in_output(output)
|
||||
|
||||
# Step 6: Save output
|
||||
effective_output_dir = output_dir !== nothing ? output_dir : "outputs/estimations/latest"
|
||||
csv_file, meta_file = save_output(output, data.metadata, segment_info, run_dir; outdir=effective_output_dir)
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("COMPLETE")
|
||||
println("="^60)
|
||||
println("Output files:")
|
||||
println(" $csv_file")
|
||||
println(" $meta_file")
|
||||
println("\nFinished: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))")
|
||||
|
||||
return output
|
||||
end
|
||||
|
||||
# Run if executed directly
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
main()
|
||||
end
|
||||
Reference in New Issue
Block a user