Document raw source file reference
This commit is contained in:
aseimel
@@ -0,0 +1,340 @@
|
||||
#!/usr/bin/env julia
|
||||
#############################################################################
|
||||
## run_model.jl
|
||||
## Main runner for latent trait model
|
||||
## Executes the complete pipeline: data loading → preparation → model fitting
|
||||
##
|
||||
## Supports two model versions:
|
||||
## - "2dim": 2D bipolar model (V1) - estimates economic_lr and galtan directly
|
||||
## - "4dim": 4D unipolar model (V10) - estimates 4 traits, derives 2 scales
|
||||
##
|
||||
## Default is 2D model (better identification, faster convergence)
|
||||
#############################################################################
|
||||
|
||||
using Dates
|
||||
|
||||
#############################################################################
|
||||
## EXECUTION CONFIGURATION (Change these values as needed)
|
||||
#############################################################################
|
||||
const MODEL_VERSION = "2dim" # "2dim" (recommended) or "4dim"
|
||||
const STAN_MODEL_FILE = MODEL_VERSION == "2dim" ? "models/stan_model_2dim_v6.stan" : "models/stan_model_4dim_v10.stan"
|
||||
const NUM_CHAINS = 4 # Number of MCMC chains (run in parallel)
|
||||
const NUM_WARMUP = 1000 # Number of warmup iterations
|
||||
const NUM_SAMPLES = 2000 # Number of sampling iterations
|
||||
const ADAPT_DELTA = 0.95 # Target acceptance probability
|
||||
const MAX_DEPTH = 15 # Maximum tree depth
|
||||
const START_YEAR = 1944 # First year to include (avoid sparse early data)
|
||||
|
||||
println("=" ^ 70)
|
||||
println("Latent Trait Model - Estimation Pipeline")
|
||||
println("=" ^ 70)
|
||||
println("Started at: ", Dates.now())
|
||||
println("Configuration:")
|
||||
println(" Model version: $(MODEL_VERSION)")
|
||||
println(" Stan model: $(STAN_MODEL_FILE)")
|
||||
println(" Chains: $(NUM_CHAINS)")
|
||||
println(" Warmup iterations: $(NUM_WARMUP)")
|
||||
println(" Sampling iterations: $(NUM_SAMPLES)")
|
||||
println(" Total iterations per chain: $(NUM_WARMUP + NUM_SAMPLES)")
|
||||
println(" Adapt delta: $(ADAPT_DELTA)")
|
||||
println(" Max depth: $(MAX_DEPTH)")
|
||||
println(" Start year: $(START_YEAR)")
|
||||
if MODEL_VERSION == "2dim"
|
||||
println("\n 2D MODEL: Estimates economic_lr and galtan directly")
|
||||
println(" (Half the parameters, better convergence)")
|
||||
else
|
||||
println("\n 4D MODEL: Estimates 4 traits, derives 2 scales")
|
||||
println(" (Known identification issues - see VERSION_HISTORY.md)")
|
||||
end
|
||||
println("=" ^ 70)
|
||||
|
||||
# Include pipeline modules
|
||||
include("pipeline/00_validation.jl") # Validation checks
|
||||
include("pipeline/02_data_loading.jl")
|
||||
include("pipeline/03_data_preparation.jl")
|
||||
include("pipeline/04_model_execution.jl")
|
||||
include("pipeline/05_results_processing.jl")
|
||||
|
||||
# Load robust save module
|
||||
include("pipeline/06_save_model.jl")
|
||||
import .RobustSave: robust_save_model
|
||||
|
||||
function run_model(;
|
||||
num_chains=NUM_CHAINS,
|
||||
num_warmup=NUM_WARMUP,
|
||||
num_samples=NUM_SAMPLES,
|
||||
adapt_delta=ADAPT_DELTA,
|
||||
max_depth=MAX_DEPTH,
|
||||
model_file=STAN_MODEL_FILE,
|
||||
start_year=START_YEAR,
|
||||
data_dir="data"
|
||||
)
|
||||
"""Run the complete latent trait model pipeline (2D or 4D based on MODEL_VERSION)"""
|
||||
|
||||
try
|
||||
# Step 1: Load and preprocess data
|
||||
println("\n" * "="^50)
|
||||
println("STEP 1: DATA LOADING")
|
||||
println("="^50)
|
||||
|
||||
manifesto, expert_dim, expert_lr, year0, union_to_constituents, constituent_to_union = load_and_preprocess_4dim_data(start_year; data_dir=data_dir)
|
||||
|
||||
# Step 2: Prepare Stan data structure
|
||||
println("\n" * "="^50)
|
||||
println("STEP 2: DATA PREPARATION")
|
||||
println("="^50)
|
||||
|
||||
# Prepare indices and mappings (V4: union-aware)
|
||||
data_prep = prepare_4dim_stan_data(manifesto, expert_dim, expert_lr, year0;
|
||||
union_to_constituents=union_to_constituents,
|
||||
constituent_to_union=constituent_to_union)
|
||||
|
||||
# Finalize Stan data dictionary (V4: includes constituent arrays)
|
||||
final_data = finalize_4dim_stan_data(
|
||||
data_prep.manifesto, data_prep.expert_dim, data_prep.expert_lr,
|
||||
data_prep.segment_year, data_prep.segment_info,
|
||||
data_prep.all_parties, data_prep.all_groups,
|
||||
data_prep.group_to_index, year0, data_prep.S, data_prep.J, data_prep.P, data_prep.R,
|
||||
data_prep.N_ciy, data_prep.len_theta_ts, data_prep.segment_country_idx,
|
||||
data_prep.F, data_prep.segment_family_idx, data_prep.anchor_segment_idx;
|
||||
N_const_man_total=data_prep.N_const_man_total,
|
||||
n_const_man=data_prep.n_const_man,
|
||||
const_offset_man=data_prep.const_offset_man,
|
||||
const_rr_man=data_prep.const_rr_man,
|
||||
N_const_exp_dim_total=data_prep.N_const_exp_dim_total,
|
||||
n_const_exp_dim=data_prep.n_const_exp_dim,
|
||||
const_offset_exp_dim=data_prep.const_offset_exp_dim,
|
||||
const_rr_exp_dim=data_prep.const_rr_exp_dim,
|
||||
N_const_exp_lr_total=data_prep.N_const_exp_lr_total,
|
||||
n_const_exp_lr=data_prep.n_const_exp_lr,
|
||||
const_offset_exp_lr=data_prep.const_offset_exp_lr,
|
||||
const_rr_exp_lr=data_prep.const_rr_exp_lr
|
||||
)
|
||||
|
||||
dat_4dim = final_data.dat_4dim
|
||||
|
||||
# Step 3: Validate data BEFORE running Stan
|
||||
println("\n" * "="^50)
|
||||
println("STEP 3: DATA VALIDATION")
|
||||
println("="^50)
|
||||
|
||||
if !validate_stan_data(dat_4dim; verbose=true)
|
||||
error("Data validation failed - see errors above")
|
||||
end
|
||||
|
||||
estimate_memory_requirements(dat_4dim; verbose=true)
|
||||
|
||||
# Step 4: Create initialization function
|
||||
println("\n" * "="^50)
|
||||
println("STEP 4: MODEL INITIALIZATION")
|
||||
println("="^50)
|
||||
|
||||
# Determine model version for initialization
|
||||
model_init_version = MODEL_VERSION == "2dim" ? "v1_2dim" : "v10"
|
||||
|
||||
# Use S (segments) for initialization, not J (parties)
|
||||
init_fn = create_init_function(dat_4dim, data_prep.S, data_prep.P,
|
||||
data_prep.R, final_data.T_year, data_prep.N_ciy;
|
||||
model_version=model_init_version)
|
||||
|
||||
# Validate initialization values
|
||||
println("\nValidating initialization for chain 1...")
|
||||
test_init = init_fn()
|
||||
if !validate_init_values(test_init; verbose=true)
|
||||
error("Initialization validation failed - see errors above")
|
||||
end
|
||||
|
||||
# Step 5: Run Stan model
|
||||
println("\n" * "="^50)
|
||||
println("STEP 5: MODEL EXECUTION")
|
||||
println("="^50)
|
||||
|
||||
# Create temp folder for output
|
||||
temp_folder = mktempdir()
|
||||
println("Temporary folder for Stan output: $temp_folder")
|
||||
|
||||
# Run Stan model
|
||||
stanmodel = run_4dim_stan_model(
|
||||
dat_4dim, init_fn, temp_folder;
|
||||
num_chains=num_chains,
|
||||
num_warmup=num_warmup,
|
||||
num_samples=num_samples,
|
||||
adapt_delta=adapt_delta,
|
||||
max_depth=max_depth,
|
||||
model_file=model_file
|
||||
)
|
||||
|
||||
# Step 6: Results Processing & Diagnostics
|
||||
println("\n" * "="^50)
|
||||
println("STEP 6: RESULTS PROCESSING & DIAGNOSTICS")
|
||||
println("="^50)
|
||||
|
||||
results = extract_model_results_4dim(stanmodel)
|
||||
diagnostics = compute_model_diagnostics(stanmodel)
|
||||
|
||||
println("\nCONVERGENCE DIAGNOSTICS:")
|
||||
println(" Max R-hat: $(round(diagnostics.max_rhat, digits=4))")
|
||||
println(" Mean R-hat: $(round(diagnostics.mean_rhat, digits=4))")
|
||||
println(" High R-hat count: $(diagnostics.high_rhat_count)")
|
||||
println(" Min ESS: $(round(diagnostics.min_ess, digits=0))")
|
||||
println(" Mean ESS: $(round(diagnostics.mean_ess, digits=0))")
|
||||
println(" Convergence Status: $(diagnostics.convergence_status)")
|
||||
|
||||
# Step 7: Save results
|
||||
println("\n" * "="^50)
|
||||
println("STEP 7: SAVING RESULTS")
|
||||
println("="^50)
|
||||
println(" Using save-local-then-move strategy...")
|
||||
|
||||
# Prepare data for saving (SINGLE copy of stanmodel, not multiple!)
|
||||
model_data_to_save = Dict{String, Any}(
|
||||
# StanModel object (contains all MCMC samples)
|
||||
"stanmodel_object" => stanmodel,
|
||||
|
||||
# Processed diagnostics
|
||||
"diagnostics_summary" => diagnostics.diagnostics_summary,
|
||||
"data_dict" => dat_4dim,
|
||||
|
||||
# Original data for reference
|
||||
"manifesto" => final_data.manifesto,
|
||||
"expert_dim" => final_data.expert_dim,
|
||||
"expert_lr" => final_data.expert_lr,
|
||||
"segment_year" => final_data.segment_year, # V10: segment-year mapping
|
||||
"segment_info" => final_data.segment_info, # V10: segment metadata (party_id, segment_num, year range)
|
||||
|
||||
# Metadata with convergence info
|
||||
"model_info" => Dict(
|
||||
"timestamp" => Dates.format(Dates.now(), "yyyy-mm-dd_HH-MM-SS"),
|
||||
"max_rhat" => diagnostics.max_rhat,
|
||||
"mean_rhat" => diagnostics.mean_rhat,
|
||||
"min_ess" => diagnostics.min_ess,
|
||||
"mean_ess" => diagnostics.mean_ess,
|
||||
"convergence_status" => diagnostics.convergence_status,
|
||||
"model_file" => model_file,
|
||||
"model_version" => MODEL_VERSION,
|
||||
"num_chains" => num_chains,
|
||||
"num_warmup" => num_warmup,
|
||||
"num_samples" => num_samples,
|
||||
"adapt_delta" => adapt_delta,
|
||||
"max_depth" => max_depth,
|
||||
"year0" => year0,
|
||||
"dimensions" => MODEL_VERSION == "2dim" ?
|
||||
["economic_lr", "galtan"] :
|
||||
["pro_market", "pro_welfare", "cosmopolitan", "traditional"]
|
||||
)
|
||||
)
|
||||
|
||||
# Save using CSV-first robust system. Chains have already been secured
|
||||
# by model execution; robust_save_model adds metadata/data and verifies.
|
||||
save_dir = data_dir != "data" ? joinpath(data_dir, "model_run") : "outputs/model_outputs"
|
||||
output_file = robust_save_model(
|
||||
stanmodel,
|
||||
model_data_to_save,
|
||||
save_dir;
|
||||
compress=true, # Ignored by CSV-first save implementation
|
||||
keep_local_backups=2 # Ignored; chains are already saved before this step
|
||||
)
|
||||
|
||||
# Robust save module already verified everything!
|
||||
|
||||
println("\n" * "="^70)
|
||||
println("MODEL EXECUTION COMPLETED SUCCESSFULLY!")
|
||||
println("="^70)
|
||||
println(" Max R-hat: $(round(diagnostics.max_rhat, digits=4))")
|
||||
println(" Mean R-hat: $(round(diagnostics.mean_rhat, digits=4))")
|
||||
println(" Convergence: $(diagnostics.convergence_status)")
|
||||
println(" Output file: $output_file")
|
||||
|
||||
# Print summary statistics
|
||||
println("\nModel Summary ($(MODEL_VERSION == "2dim" ? "2D Direct Bipolar" : "4D Unipolar")):")
|
||||
println(" Segments: $(data_prep.S)")
|
||||
println(" Parties with valid segments: $(data_prep.J)")
|
||||
println(" Countries: $(data_prep.P)")
|
||||
println(" Segment-year combinations: $(data_prep.R)")
|
||||
println(" Years: $(final_data.T_year)")
|
||||
println(" Manifesto observations: $(dat_4dim["N_man"])")
|
||||
println(" Expert dimension-specific observations: $(dat_4dim["N_exp_dim"])")
|
||||
println(" Expert general L-R observations: $(dat_4dim["N_exp_lr"])")
|
||||
if MODEL_VERSION == "2dim"
|
||||
println(" Dimensions estimated: 2 (economic_lr, galtan)")
|
||||
println(" Theta parameters: $(2 * data_prep.R) (2 × R)")
|
||||
else
|
||||
println(" Dimensions estimated: 4 (pro_market, pro_welfare, cosmopolitan, traditional)")
|
||||
println(" Theta parameters: $(4 * data_prep.R) (4 × R)")
|
||||
end
|
||||
println("=" ^ 70)
|
||||
|
||||
# Cleanup temp folder after successful save
|
||||
println("\nCLEANING UP TEMPORARY FILES...")
|
||||
try
|
||||
if isdir(temp_folder)
|
||||
rm(temp_folder, recursive=true, force=true)
|
||||
println(" Removed temporary folder: $temp_folder")
|
||||
end
|
||||
catch cleanup_error
|
||||
println(" Warning: Could not remove temp folder: $cleanup_error")
|
||||
println(" (This won't affect your saved results)")
|
||||
end
|
||||
|
||||
return true
|
||||
|
||||
catch e
|
||||
println("\nERROR in model pipeline: $e")
|
||||
println("Stack trace:")
|
||||
showerror(stdout, e, catch_backtrace())
|
||||
rethrow(e)
|
||||
end
|
||||
end
|
||||
|
||||
function main(args=ARGS)
|
||||
# Parse --data-dir argument
|
||||
data_dir = "data"
|
||||
for (i, arg) in enumerate(args)
|
||||
if arg == "--data-dir" && i < length(args)
|
||||
data_dir = args[i + 1]
|
||||
elseif startswith(arg, "--data-dir=")
|
||||
data_dir = split(arg, "=", limit=2)[2]
|
||||
end
|
||||
end
|
||||
|
||||
if data_dir != "."
|
||||
println("Using data directory: $data_dir")
|
||||
end
|
||||
println("Executing $(MODEL_VERSION) latent trait model pipeline...")
|
||||
|
||||
# Check that required data files exist (in data_dir)
|
||||
required_data = [joinpath(data_dir, f) for f in ["text_data.csv", "expert.csv", "lr_data.csv"]]
|
||||
required_files = vcat(required_data, [STAN_MODEL_FILE])
|
||||
missing_files = []
|
||||
|
||||
for file in required_files
|
||||
if !isfile(file)
|
||||
push!(missing_files, file)
|
||||
end
|
||||
end
|
||||
|
||||
if !isempty(missing_files)
|
||||
println("ERROR: Missing required files:")
|
||||
for file in missing_files
|
||||
println(" - $file")
|
||||
end
|
||||
|
||||
if any(f -> endswith(f, "text_data.csv") || endswith(f, "expert.csv") || endswith(f, "lr_data.csv"), missing_files)
|
||||
println("\nTo generate data files, run:")
|
||||
println(" bash scripts/01_prepare_data.sh")
|
||||
end
|
||||
|
||||
error("Cannot proceed without required files")
|
||||
end
|
||||
|
||||
# Run the complete pipeline
|
||||
results = run_model(data_dir=data_dir)
|
||||
|
||||
println("\n$(MODEL_VERSION) latent trait model pipeline completed successfully!")
|
||||
println("Check outputs/model_outputs/latest/ for chain CSVs and metadata.")
|
||||
end
|
||||
|
||||
# Main execution
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
main()
|
||||
end
|
||||
@@ -0,0 +1,211 @@
|
||||
#!/usr/bin/env julia
|
||||
#=
|
||||
02_enrich_output.jl - Enrich party positions CSV with metadata
|
||||
|
||||
Fast post-processing script that adds party names, union membership status,
|
||||
and election results to the model output CSV. Operates purely on CSV files
|
||||
(no chain loading). Takes seconds, not minutes.
|
||||
|
||||
Usage:
|
||||
julia 02_enrich_output.jl # enriches latest party_positions_*.csv
|
||||
julia 02_enrich_output.jl somefile.csv # enriches a specific file
|
||||
|
||||
Adds columns:
|
||||
party_name_english - Full English party name (from data/party_names.csv)
|
||||
party_name_short - Standard abbreviation
|
||||
in_union - 1 if party's union had a joint manifesto that year, 0 otherwise
|
||||
pervote - Vote share (%) at election years; missing for non-election years
|
||||
=#
|
||||
|
||||
using CSV
|
||||
using DataFrames
|
||||
using Dates
|
||||
|
||||
function find_latest_output()
|
||||
outdir = "outputs/estimations/latest"
|
||||
files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
|
||||
!contains(f, "metadata") && !contains(f, "tables"),
|
||||
readdir(outdir))
|
||||
isempty(files) && error("No party_positions_*.csv found in $outdir/")
|
||||
sort!(files, rev=true)
|
||||
return joinpath(outdir, files[1])
|
||||
end
|
||||
|
||||
function enrich(input_file::String)
|
||||
println("="^60)
|
||||
println("ENRICH OUTPUT")
|
||||
println("="^60)
|
||||
println("Input: $input_file")
|
||||
|
||||
output = CSV.read(input_file, DataFrame)
|
||||
println(" Rows: $(nrow(output)), Columns: $(ncol(output))")
|
||||
|
||||
# --- Party names ---
|
||||
party_names_file = joinpath("data", "party_names.csv")
|
||||
if isfile(party_names_file)
|
||||
names_df = CSV.read(party_names_file, DataFrame)
|
||||
name_lookup = Dict{Int, Tuple{String, String}}()
|
||||
for row in eachrow(names_df)
|
||||
short = ismissing(row.party_name_short) ? "" : string(row.party_name_short)
|
||||
name_lookup[row.partyfacts_id] = (string(row.party_name_english), short)
|
||||
end
|
||||
output.party_name_english = [
|
||||
haskey(name_lookup, pid) ? name_lookup[pid][1] : ""
|
||||
for pid in output.party_id
|
||||
]
|
||||
output.party_name_short = [
|
||||
haskey(name_lookup, pid) ? name_lookup[pid][2] : ""
|
||||
for pid in output.party_id
|
||||
]
|
||||
n_named = count(x -> x != "", output.party_name_english)
|
||||
println(" Party names: $n_named / $(nrow(output)) rows matched")
|
||||
else
|
||||
output.party_name_english = fill("", nrow(output))
|
||||
output.party_name_short = fill("", nrow(output))
|
||||
println(" WARNING: data/party_names.csv not found")
|
||||
end
|
||||
|
||||
# --- Union mapping (for in_union + pervote fallback) ---
|
||||
union_mapping_file = joinpath("data", "union_mapping.csv")
|
||||
constituent_to_union = Dict{Int, Int}()
|
||||
if isfile(union_mapping_file)
|
||||
union_df = CSV.read(union_mapping_file, DataFrame)
|
||||
for row in eachrow(union_df)
|
||||
constituent_to_union[row.expert_pf_id] = row.manifesto_pf_id
|
||||
end
|
||||
end
|
||||
|
||||
# --- in_union dummy (year-varying) ---
|
||||
text_data_file = "data/text_data.csv"
|
||||
output.in_union = zeros(Int, nrow(output))
|
||||
|
||||
if isfile(text_data_file) && !isempty(constituent_to_union)
|
||||
text_df = CSV.read(text_data_file, DataFrame)
|
||||
|
||||
# Build set of (party_pf_id, year) pairs for manifesto data
|
||||
manifesto_text = filter(r -> r.project == "Manifesto Project", text_df)
|
||||
manifesto_party_years = Set{Tuple{Int, Int}}()
|
||||
for row in eachrow(manifesto_text)
|
||||
push!(manifesto_party_years, (row.party, row.year))
|
||||
end
|
||||
|
||||
# Process each (party, segment) group
|
||||
gdf = groupby(output, [:party_id, :segment_num])
|
||||
for subdf in gdf
|
||||
pid = subdf.party_id[1]
|
||||
!haskey(constituent_to_union, pid) && continue
|
||||
union_id = constituent_to_union[pid]
|
||||
|
||||
# Get row indices in the full output for this group
|
||||
idxs = parentindices(subdf)[1]
|
||||
|
||||
# Determine in_union at election years
|
||||
election_year_vals = Dict{Int, Int}()
|
||||
for (j, row) in enumerate(eachrow(subdf))
|
||||
if (union_id, row.year) in manifesto_party_years
|
||||
election_year_vals[row.year] = 1
|
||||
elseif (pid, row.year) in manifesto_party_years
|
||||
election_year_vals[row.year] = 0
|
||||
end
|
||||
end
|
||||
|
||||
# Forward-fill within segment
|
||||
sorted_pairs = sort(collect(zip(subdf.year, idxs)))
|
||||
last_val = 0
|
||||
for (yr, idx) in sorted_pairs
|
||||
if haskey(election_year_vals, yr)
|
||||
last_val = election_year_vals[yr]
|
||||
end
|
||||
output.in_union[idx] = last_val
|
||||
end
|
||||
end
|
||||
|
||||
n_in_union = count(x -> x == 1, output.in_union)
|
||||
println(" in_union: $n_in_union rows flagged as union members")
|
||||
else
|
||||
println(" WARNING: Could not compute in_union (missing files)")
|
||||
end
|
||||
|
||||
# --- Election results (pervote) ---
|
||||
election_file = joinpath("data", "election_data.csv")
|
||||
output.pervote = Vector{Union{Float64, Missing}}(missing, nrow(output))
|
||||
|
||||
if isfile(election_file)
|
||||
election_df = CSV.read(election_file, DataFrame)
|
||||
|
||||
# Build lookup: (party_id, year) -> pervote
|
||||
election_lookup = Dict{Tuple{Int, Int}, Float64}()
|
||||
for row in eachrow(election_df)
|
||||
election_lookup[(row.party, row.year)] = row.pervote
|
||||
end
|
||||
|
||||
n_filled = 0
|
||||
for i in 1:nrow(output)
|
||||
pid = output.party_id[i]
|
||||
yr = output.year[i]
|
||||
|
||||
# Direct match (standalone party)
|
||||
if haskey(election_lookup, (pid, yr))
|
||||
output.pervote[i] = election_lookup[(pid, yr)]
|
||||
n_filled += 1
|
||||
elseif hasproperty(output, :union_party_id) && !ismissing(output.union_party_id[i])
|
||||
# Union constituent: look up by union PF ID
|
||||
uid = output.union_party_id[i]
|
||||
if haskey(election_lookup, (uid, yr))
|
||||
output.pervote[i] = election_lookup[(uid, yr)]
|
||||
n_filled += 1
|
||||
end
|
||||
elseif haskey(constituent_to_union, pid)
|
||||
# Fallback: use union mapping even if union_party_id column not populated
|
||||
uid = constituent_to_union[pid]
|
||||
if haskey(election_lookup, (uid, yr))
|
||||
output.pervote[i] = election_lookup[(uid, yr)]
|
||||
n_filled += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
println(" pervote: $n_filled / $(nrow(output)) rows filled")
|
||||
else
|
||||
println(" WARNING: election_data.csv not found")
|
||||
end
|
||||
|
||||
# --- Reorder columns ---
|
||||
estimate_cols = Symbol[]
|
||||
for base in ["economic_lr", "galtan", "pro_market", "pro_welfare", "cosmopolitan", "traditional"]
|
||||
sym = Symbol(base)
|
||||
if hasproperty(output, sym)
|
||||
push!(estimate_cols, sym)
|
||||
push!(estimate_cols, Symbol("$(base)_se"))
|
||||
push!(estimate_cols, Symbol("$(base)_q025"))
|
||||
push!(estimate_cols, Symbol("$(base)_q975"))
|
||||
end
|
||||
end
|
||||
|
||||
# Fix country code: MO (Macau) → MK (North Macedonia) — GPS uses wrong ISO2
|
||||
output.country = replace(output.country, "MO" => "MK")
|
||||
|
||||
col_order = vcat(
|
||||
[:party_id, :party_name_english, :party_name_short, :country, :year, :segment_num,
|
||||
:union_party_id, :in_union, :pervote],
|
||||
estimate_cols
|
||||
)
|
||||
col_order = filter(c -> hasproperty(output, c), col_order)
|
||||
select!(output, col_order)
|
||||
|
||||
# --- Write back ---
|
||||
CSV.write(input_file, output)
|
||||
println("\n Wrote: $input_file")
|
||||
println(" Columns ($(ncol(output))): $(join(string.(names(output)), ", "))")
|
||||
|
||||
return output
|
||||
end
|
||||
|
||||
function main(args=ARGS)
|
||||
input = length(args) >= 1 ? args[1] : find_latest_output()
|
||||
enrich(input)
|
||||
println("\nDone.")
|
||||
end
|
||||
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
main()
|
||||
end
|
||||
@@ -0,0 +1,985 @@
|
||||
#!/usr/bin/env julia
|
||||
#=
|
||||
02_post_estimation.jl - Extract party position estimates from Stan model output
|
||||
|
||||
Supports both model versions:
|
||||
- 2D model (V1): Extracts economic_lr and galtan directly
|
||||
- 4D model (V10): Extracts 4 traits + 2 derived scales
|
||||
|
||||
V10/V1 UPDATE: Handles segment-based indexing
|
||||
- Maps segment results back to original party IDs
|
||||
- Adds segment_num column to indicate which segment of the party
|
||||
- Flags discontinuities for parties with multiple segments
|
||||
|
||||
This script:
|
||||
1. Auto-detects the latest model run in model_outputs/
|
||||
2. Loads the chain CSV files and data mappings
|
||||
3. Detects model version from metadata or column names
|
||||
4. Extracts posterior summaries for all segment-year positions
|
||||
5. Maps Stan parameter indices back to real party IDs, segment numbers, and years
|
||||
6. Saves output as wide-format CSV with uncertainty estimates
|
||||
|
||||
Usage:
|
||||
julia 02_post_estimation.jl
|
||||
|
||||
Output:
|
||||
party_positions_YYYY-MM-DD_HH-MM-SS.csv
|
||||
=#
|
||||
|
||||
using CSV
|
||||
using DataFrames
|
||||
using Statistics
|
||||
using JSON
|
||||
using Dates
|
||||
using Printf
|
||||
|
||||
# =============================================================================
|
||||
# STEP 0: Auto-detect latest run
|
||||
# =============================================================================
|
||||
|
||||
function find_latest_run(base_dir::String="outputs/model_outputs/latest")
|
||||
if !isdir(base_dir)
|
||||
error("Model outputs directory not found: $base_dir")
|
||||
end
|
||||
|
||||
runs = filter(d -> startswith(d, "run_") && isdir(joinpath(base_dir, d)), readdir(base_dir))
|
||||
|
||||
if isempty(runs)
|
||||
error("No runs found in $base_dir")
|
||||
end
|
||||
|
||||
# Sort by timestamp in directory name (format: run_YYYY-MM-DD_HH-MM-SS)
|
||||
sort!(runs, rev=true)
|
||||
|
||||
latest = joinpath(base_dir, runs[1])
|
||||
println("Found $(length(runs)) run(s). Using latest: $latest")
|
||||
return latest
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 1: Load data and build segment-year lookup
|
||||
# =============================================================================
|
||||
|
||||
function load_run_data(run_dir::String)
|
||||
println("\n" * "="^60)
|
||||
println("LOADING RUN DATA")
|
||||
println("="^60)
|
||||
|
||||
data_dir = joinpath(run_dir, "data")
|
||||
chains_dir = joinpath(run_dir, "chains")
|
||||
|
||||
# Check required files exist
|
||||
required_files = [
|
||||
joinpath(data_dir, "text_data.csv"),
|
||||
joinpath(data_dir, "expert_dim.csv"),
|
||||
joinpath(data_dir, "expert_lr.csv"),
|
||||
joinpath(run_dir, "metadata.json")
|
||||
]
|
||||
|
||||
for f in required_files
|
||||
if !isfile(f)
|
||||
error("Required file not found: $f")
|
||||
end
|
||||
end
|
||||
|
||||
# Load data files
|
||||
println("Loading text_data.csv...")
|
||||
text_data = CSV.read(joinpath(data_dir, "text_data.csv"), DataFrame)
|
||||
println(" Rows: $(nrow(text_data))")
|
||||
|
||||
println("Loading expert_dim.csv...")
|
||||
expert_dim = CSV.read(joinpath(data_dir, "expert_dim.csv"), DataFrame)
|
||||
println(" Rows: $(nrow(expert_dim))")
|
||||
|
||||
println("Loading expert_lr.csv...")
|
||||
expert_lr = CSV.read(joinpath(data_dir, "expert_lr.csv"), DataFrame)
|
||||
println(" Rows: $(nrow(expert_lr))")
|
||||
|
||||
println("Loading metadata.json...")
|
||||
metadata = JSON.parsefile(joinpath(run_dir, "metadata.json"))
|
||||
println(" year0: $(metadata["year0"])")
|
||||
println(" Model: $(metadata["model_file"])")
|
||||
|
||||
# V10: Load segment_info if available
|
||||
segment_info_file = joinpath(data_dir, "segment_info.csv")
|
||||
segment_info = nothing
|
||||
if isfile(segment_info_file)
|
||||
println("Loading segment_info.csv (V10)...")
|
||||
segment_info = CSV.read(segment_info_file, DataFrame)
|
||||
println(" Segments: $(nrow(segment_info))")
|
||||
end
|
||||
|
||||
# V10: Load segment_year_map if available
|
||||
segment_year_file = joinpath(data_dir, "segment_year_map.csv")
|
||||
segment_year_map = nothing
|
||||
if isfile(segment_year_file)
|
||||
println("Loading segment_year_map.csv (V10)...")
|
||||
segment_year_map = CSV.read(segment_year_file, DataFrame)
|
||||
println(" Segment-years: $(nrow(segment_year_map))")
|
||||
end
|
||||
|
||||
# Find chain files
|
||||
chain_files = filter(f -> endswith(f, ".csv") && startswith(f, "chain_"), readdir(chains_dir))
|
||||
println("\nFound $(length(chain_files)) chain file(s)")
|
||||
|
||||
return (
|
||||
text_data = text_data,
|
||||
expert_dim = expert_dim,
|
||||
expert_lr = expert_lr,
|
||||
metadata = metadata,
|
||||
segment_info = segment_info,
|
||||
segment_year_map = segment_year_map,
|
||||
chain_files = [joinpath(chains_dir, f) for f in sort(chain_files)],
|
||||
run_dir = run_dir
|
||||
)
|
||||
end
|
||||
|
||||
function normalize_country_value(value)
|
||||
if ismissing(value)
|
||||
return missing
|
||||
end
|
||||
txt = strip(string(value))
|
||||
return isempty(txt) ? missing : txt
|
||||
end
|
||||
|
||||
function build_party_country_map(text_data::DataFrame, expert_dim::DataFrame, expert_lr::DataFrame)
|
||||
merged = unique(vcat(
|
||||
select(text_data, :party, :country),
|
||||
select(expert_dim, :party, :country),
|
||||
select(expert_lr, :party, :country)
|
||||
))
|
||||
|
||||
party_to_country = Dict{Int, String}()
|
||||
for row in eachrow(merged)
|
||||
pid = tryparse(Int, string(row.party))
|
||||
if pid === nothing
|
||||
continue
|
||||
end
|
||||
c = normalize_country_value(row.country)
|
||||
if !ismissing(c)
|
||||
party_to_country[pid] = c
|
||||
end
|
||||
end
|
||||
return party_to_country
|
||||
end
|
||||
|
||||
function load_constituent_to_union_map()::Dict{Int, Int}
|
||||
mapping_file = joinpath("data", "union_mapping.csv")
|
||||
constituent_to_union = Dict{Int, Int}()
|
||||
if isfile(mapping_file)
|
||||
union_df = CSV.read(mapping_file, DataFrame)
|
||||
for row in eachrow(union_df)
|
||||
constituent_to_union[row.expert_pf_id] = row.manifesto_pf_id
|
||||
end
|
||||
end
|
||||
return constituent_to_union
|
||||
end
|
||||
|
||||
function resolve_party_country(pid_value,
|
||||
party_to_country::Dict{Int, String},
|
||||
constituent_to_union::Dict{Int, Int})
|
||||
pid = tryparse(Int, string(pid_value))
|
||||
if pid === nothing
|
||||
return missing, "unresolved"
|
||||
end
|
||||
|
||||
if haskey(party_to_country, pid)
|
||||
return party_to_country[pid], "direct"
|
||||
end
|
||||
|
||||
if haskey(constituent_to_union, pid)
|
||||
uid = constituent_to_union[pid]
|
||||
if haskey(party_to_country, uid)
|
||||
return party_to_country[uid], "union_fallback"
|
||||
end
|
||||
end
|
||||
|
||||
return missing, "unresolved"
|
||||
end
|
||||
|
||||
function apply_country_resolution!(df::DataFrame,
|
||||
party_col::Symbol,
|
||||
country_col::Symbol,
|
||||
party_to_country::Dict{Int, String},
|
||||
constituent_to_union::Dict{Int, Int})
|
||||
resolved_country = Union{Missing, String}[]
|
||||
source_counts = Dict("direct" => 0, "union_fallback" => 0, "unresolved" => 0)
|
||||
unresolved_parties = Set{Int}()
|
||||
|
||||
for pid in df[!, party_col]
|
||||
country, source = resolve_party_country(pid, party_to_country, constituent_to_union)
|
||||
push!(resolved_country, country)
|
||||
source_counts[source] += 1
|
||||
if source == "unresolved"
|
||||
pid_int = tryparse(Int, string(pid))
|
||||
if pid_int !== nothing
|
||||
push!(unresolved_parties, pid_int)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
df[!, country_col] = resolved_country
|
||||
return source_counts, sort!(collect(unresolved_parties))
|
||||
end
|
||||
|
||||
function fill_missing_countries!(df::DataFrame,
|
||||
segment_info::Union{DataFrame, Nothing},
|
||||
party_to_country::Dict{Int, String},
|
||||
constituent_to_union::Dict{Int, Int})
|
||||
if !hasproperty(df, :country)
|
||||
source_counts, unresolved = apply_country_resolution!(
|
||||
df, :party_id, :country, party_to_country, constituent_to_union
|
||||
)
|
||||
return source_counts, unresolved
|
||||
end
|
||||
|
||||
normalized = Union{Missing, String}[]
|
||||
for val in df.country
|
||||
push!(normalized, normalize_country_value(val))
|
||||
end
|
||||
df.country = normalized
|
||||
|
||||
source_counts = Dict("direct" => 0, "union_fallback" => 0, "segment_info" => 0, "unresolved" => 0)
|
||||
|
||||
segment_country_by_id = Dict{Int, String}()
|
||||
if segment_info !== nothing && hasproperty(segment_info, :country)
|
||||
for row in eachrow(segment_info)
|
||||
c = normalize_country_value(row.country)
|
||||
if !ismissing(c)
|
||||
segment_country_by_id[Int(row.segment_id)] = c
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
unresolved_parties = Set{Int}()
|
||||
for i in 1:nrow(df)
|
||||
if !ismissing(df.country[i])
|
||||
continue
|
||||
end
|
||||
|
||||
if hasproperty(df, :segment_id) && haskey(segment_country_by_id, Int(df.segment_id[i]))
|
||||
df.country[i] = segment_country_by_id[Int(df.segment_id[i])]
|
||||
source_counts["segment_info"] += 1
|
||||
continue
|
||||
end
|
||||
|
||||
country, source = resolve_party_country(df.party_id[i], party_to_country, constituent_to_union)
|
||||
if !ismissing(country)
|
||||
df.country[i] = country
|
||||
source_counts[source] += 1
|
||||
else
|
||||
source_counts["unresolved"] += 1
|
||||
pid_int = tryparse(Int, string(df.party_id[i]))
|
||||
pid_int !== nothing && push!(unresolved_parties, pid_int)
|
||||
end
|
||||
end
|
||||
|
||||
return source_counts, sort!(collect(unresolved_parties))
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 2: Build complete segment-year mapping (V10) or party-year mapping (V9)
|
||||
# =============================================================================
|
||||
|
||||
function build_segment_year_map(text_data::DataFrame, expert_dim::DataFrame, expert_lr::DataFrame,
|
||||
segment_info::Union{DataFrame, Nothing},
|
||||
segment_year_map::Union{DataFrame, Nothing},
|
||||
run_dir::String,
|
||||
year0::Int)
|
||||
println("\n" * "="^60)
|
||||
println("BUILDING SEGMENT-YEAR MAPPING")
|
||||
println("="^60)
|
||||
|
||||
party_to_country = build_party_country_map(text_data, expert_dim, expert_lr)
|
||||
constituent_to_union = load_constituent_to_union_map()
|
||||
|
||||
# V10: Use segment_year_map if available
|
||||
if segment_year_map !== nothing && segment_info !== nothing
|
||||
println("Using segment_year_map.csv (V10 mode)")
|
||||
|
||||
# Convert relative Year to absolute year
|
||||
if hasproperty(segment_year_map, :Year)
|
||||
segment_year_map.year = segment_year_map.Year .+ year0
|
||||
elseif !hasproperty(segment_year_map, :year)
|
||||
error("segment_year_map has no Year or year column")
|
||||
end
|
||||
|
||||
# Add party_id from segment_info if not already present
|
||||
if !hasproperty(segment_year_map, :party_id)
|
||||
segment_id_to_party = Dict(row.segment_id => row.party_id for row in eachrow(segment_info))
|
||||
segment_year_map.party_id = [segment_id_to_party[sid] for sid in segment_year_map.segment_id]
|
||||
end
|
||||
|
||||
# Add segment_num from segment_info if not already present
|
||||
if !hasproperty(segment_year_map, :segment_num)
|
||||
segment_id_to_segnum = Dict(row.segment_id => row.segment_num for row in eachrow(segment_info))
|
||||
segment_year_map.segment_num = [segment_id_to_segnum[sid] for sid in segment_year_map.segment_id]
|
||||
end
|
||||
|
||||
# Resolve/fill country column using segment metadata first, then direct and union-fallback lookup.
|
||||
source_counts, unresolved = fill_missing_countries!(
|
||||
segment_year_map, segment_info, party_to_country, constituent_to_union
|
||||
)
|
||||
direct_count = get(source_counts, "direct", 0)
|
||||
union_count = get(source_counts, "union_fallback", 0)
|
||||
segment_info_count = get(source_counts, "segment_info", 0)
|
||||
unresolved_count = count(ismissing, segment_year_map.country)
|
||||
println(" Country resolution fill counts: direct=$direct_count, union_fallback=$union_count, segment_info=$segment_info_count, unresolved_rows=$unresolved_count")
|
||||
if !isempty(unresolved)
|
||||
println(" Warning: unresolved country party IDs (first 20): $(unresolved[1:min(20, length(unresolved))])")
|
||||
end
|
||||
|
||||
R = maximum(segment_year_map.rr)
|
||||
n_segments = length(unique(segment_year_map.segment_id))
|
||||
n_parties = length(unique(segment_year_map.party_id))
|
||||
|
||||
println("Loaded segment_year_map: $(nrow(segment_year_map)) segment-years (R=$R)")
|
||||
println(" Unique segments: $n_segments")
|
||||
println(" Unique parties: $n_parties")
|
||||
|
||||
# Count observed vs interpolated
|
||||
observed_rrs = Set{Int}()
|
||||
if hasproperty(text_data, :rr_man)
|
||||
union!(observed_rrs, Set(text_data.rr_man))
|
||||
end
|
||||
if hasproperty(expert_dim, :rr_exp_dim)
|
||||
union!(observed_rrs, Set(expert_dim.rr_exp_dim))
|
||||
end
|
||||
if hasproperty(expert_lr, :rr_exp_lr)
|
||||
union!(observed_rrs, Set(expert_lr.rr_exp_lr))
|
||||
end
|
||||
|
||||
n_observed = length(intersect(Set(segment_year_map.rr), observed_rrs))
|
||||
n_interpolated = nrow(segment_year_map) - n_observed
|
||||
println(" Observed segment-years: $n_observed")
|
||||
println(" Interpolated segment-years: $n_interpolated")
|
||||
|
||||
return segment_year_map, R, segment_info
|
||||
end
|
||||
|
||||
# V9 fallback: Use party_year_map
|
||||
party_year_file = joinpath(run_dir, "data", "party_year_map.csv")
|
||||
if isfile(party_year_file)
|
||||
println("Loading party_year_map.csv (V9 fallback mode)")
|
||||
party_year_map = CSV.read(party_year_file, DataFrame)
|
||||
|
||||
# Add party_id column (same as party for V9)
|
||||
if !hasproperty(party_year_map, :party_id)
|
||||
party_year_map.party_id = party_year_map.party
|
||||
end
|
||||
|
||||
# Add segment_num column (always 1 for V9)
|
||||
if !hasproperty(party_year_map, :segment_num)
|
||||
party_year_map.segment_num = ones(Int, nrow(party_year_map))
|
||||
end
|
||||
|
||||
# Add segment_id column (same as party index for V9)
|
||||
if !hasproperty(party_year_map, :segment_id)
|
||||
party_year_map.segment_id = party_year_map.party
|
||||
end
|
||||
|
||||
# Convert relative Year to absolute year
|
||||
if hasproperty(party_year_map, :Year)
|
||||
party_year_map.year = party_year_map.Year .+ year0
|
||||
elseif !hasproperty(party_year_map, :year)
|
||||
error("party_year_map has no Year or year column")
|
||||
end
|
||||
|
||||
# Resolve/fill country column
|
||||
if !hasproperty(party_year_map, :country)
|
||||
source_counts, unresolved = apply_country_resolution!(
|
||||
party_year_map, :party_id, :country, party_to_country, constituent_to_union
|
||||
)
|
||||
direct_count = source_counts["direct"]
|
||||
union_count = source_counts["union_fallback"]
|
||||
unresolved_count = source_counts["unresolved"]
|
||||
println(" Country resolution sources: direct=$direct_count, union_fallback=$union_count, unresolved=$unresolved_count")
|
||||
if !isempty(unresolved)
|
||||
println(" Warning: unresolved country party IDs (first 20): $(unresolved[1:min(20, length(unresolved))])")
|
||||
end
|
||||
else
|
||||
normalized = Union{Missing, String}[]
|
||||
for val in party_year_map.country
|
||||
push!(normalized, normalize_country_value(val))
|
||||
end
|
||||
party_year_map.country = normalized
|
||||
end
|
||||
|
||||
R = maximum(party_year_map.rr)
|
||||
println("Loaded party_year_map: $(nrow(party_year_map)) party-years (R=$R)")
|
||||
|
||||
return party_year_map, R, nothing
|
||||
end
|
||||
|
||||
# Fallback: Reconstruct from data files
|
||||
@warn "No mapping file found, reconstructing from data (observed years only)"
|
||||
|
||||
# Extract unique party-year-rr combinations from text_data
|
||||
text_map = unique(select(text_data, :party, :country, :year, :rr_man))
|
||||
rename!(text_map, :rr_man => :rr)
|
||||
text_map.party_id = text_map.party
|
||||
text_map.segment_num = ones(Int, nrow(text_map))
|
||||
|
||||
expert_dim_map = unique(select(expert_dim, :party, :country, :year, :rr_exp_dim))
|
||||
rename!(expert_dim_map, :rr_exp_dim => :rr)
|
||||
expert_dim_map.party_id = expert_dim_map.party
|
||||
expert_dim_map.segment_num = ones(Int, nrow(expert_dim_map))
|
||||
|
||||
expert_lr_map = unique(select(expert_lr, :party, :country, :year, :rr_exp_lr))
|
||||
rename!(expert_lr_map, :rr_exp_lr => :rr)
|
||||
expert_lr_map.party_id = expert_lr_map.party
|
||||
expert_lr_map.segment_num = ones(Int, nrow(expert_lr_map))
|
||||
|
||||
combined = vcat(text_map, expert_dim_map, expert_lr_map)
|
||||
segment_year_map = unique(combined)
|
||||
sort!(segment_year_map, :rr)
|
||||
|
||||
R = maximum(segment_year_map.rr)
|
||||
println("Reconstructed mapping: $(nrow(segment_year_map)) segment-years (R=$R)")
|
||||
|
||||
return segment_year_map, R, nothing
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 3: Load and combine chains
|
||||
# =============================================================================
|
||||
|
||||
function load_chains(chain_files::Vector{String})
|
||||
println("\n" * "="^60)
|
||||
println("LOADING STAN CHAINS")
|
||||
println("="^60)
|
||||
flush(stdout)
|
||||
|
||||
chains = DataFrame[]
|
||||
|
||||
# The full Stan CSVs are very wide (hundreds of thousands of columns). For
|
||||
# post-estimation we only need party-position generated quantities. Reading
|
||||
# all columns can take hours and allocate many GB of irrelevant parameters.
|
||||
post_estimation_prefixes = (
|
||||
"economic_lr.",
|
||||
"galtan.",
|
||||
"pro_market.",
|
||||
"pro_welfare.",
|
||||
"cosmopolitan.",
|
||||
"traditional.",
|
||||
)
|
||||
keep_post_estimation_col(_i, name) = any(startswith(String(name), p) for p in post_estimation_prefixes)
|
||||
|
||||
for (i, f) in enumerate(chain_files)
|
||||
println("Loading chain $i: $(basename(f))...")
|
||||
flush(stdout)
|
||||
# Skip comment lines (Stan header) and parse only needed quantities.
|
||||
chain = CSV.read(f, DataFrame; comment="#", select=keep_post_estimation_col)
|
||||
println(" Samples: $(nrow(chain)), Parameters: $(ncol(chain))")
|
||||
flush(stdout)
|
||||
push!(chains, chain)
|
||||
end
|
||||
|
||||
# Combine chains
|
||||
println("Combining selected chain columns...")
|
||||
flush(stdout)
|
||||
combined = vcat(chains...)
|
||||
println("\nCombined: $(nrow(combined)) total samples")
|
||||
println("Selected parameters: $(ncol(combined))")
|
||||
flush(stdout)
|
||||
|
||||
return combined
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 4: Extract generated quantities
|
||||
# =============================================================================
|
||||
|
||||
"""
|
||||
Detect model version from chain column names.
|
||||
Returns "2dim" or "4dim".
|
||||
"""
|
||||
function detect_model_version(chains::DataFrame)
|
||||
cols = names(chains)
|
||||
# 2D model has economic_lr but NOT pro_market
|
||||
has_economic_lr = any(c -> startswith(string(c), "economic_lr."), cols)
|
||||
has_pro_market = any(c -> startswith(string(c), "pro_market."), cols)
|
||||
|
||||
if has_economic_lr && !has_pro_market
|
||||
return "2dim"
|
||||
elseif has_pro_market
|
||||
return "4dim"
|
||||
else
|
||||
error("Could not detect model version from chain columns")
|
||||
end
|
||||
end
|
||||
|
||||
function extract_estimates(chains::DataFrame, segment_year_map::DataFrame, R::Int)
|
||||
println("\n" * "="^60)
|
||||
println("EXTRACTING POSTERIOR ESTIMATES")
|
||||
println("="^60)
|
||||
|
||||
# Auto-detect model version from columns
|
||||
model_version = detect_model_version(chains)
|
||||
println("Detected model version: $model_version")
|
||||
|
||||
# Select quantities based on model version
|
||||
if model_version == "2dim"
|
||||
# 2D model: economic_lr and galtan are directly estimated
|
||||
# (general_lr is computed in Stan for anchoring but not extracted as output)
|
||||
quantities = ["economic_lr", "galtan"]
|
||||
test_col = "economic_lr.1"
|
||||
else
|
||||
# 4D model: 4 traits + 2 derived scales
|
||||
quantities = ["pro_market", "pro_welfare", "cosmopolitan", "traditional", "economic_lr", "galtan"]
|
||||
test_col = "pro_market.1"
|
||||
end
|
||||
|
||||
# Check that columns exist
|
||||
if !hasproperty(chains, Symbol(test_col))
|
||||
error("Column $test_col not found in chains. Available columns: $(first(names(chains), 10))...")
|
||||
end
|
||||
|
||||
n_samples = nrow(chains)
|
||||
println("Samples per parameter: $n_samples")
|
||||
|
||||
# Load union mapping for adding union_party_id column
|
||||
union_mapping_file = joinpath("data", "union_mapping.csv")
|
||||
constituent_to_union_pf = Dict{Int, Int}()
|
||||
if isfile(union_mapping_file)
|
||||
union_df = CSV.read(union_mapping_file, DataFrame)
|
||||
for row in eachrow(union_df)
|
||||
constituent_to_union_pf[row.expert_pf_id] = row.manifesto_pf_id
|
||||
end
|
||||
end
|
||||
|
||||
# Pre-allocate output DataFrame
|
||||
n_rows = nrow(segment_year_map)
|
||||
|
||||
# Add union_party_id column: NA for standalone parties, union PF ID for constituents
|
||||
union_ids = Union{Int, Missing}[]
|
||||
for pid in segment_year_map.party_id
|
||||
pid_int = isa(pid, Integer) ? pid : tryparse(Int, string(pid))
|
||||
if pid_int !== nothing && haskey(constituent_to_union_pf, pid_int)
|
||||
push!(union_ids, constituent_to_union_pf[pid_int])
|
||||
else
|
||||
push!(union_ids, missing)
|
||||
end
|
||||
end
|
||||
|
||||
output = DataFrame(
|
||||
party_id = segment_year_map.party_id,
|
||||
union_party_id = union_ids,
|
||||
segment_num = segment_year_map.segment_num,
|
||||
country = segment_year_map.country,
|
||||
year = segment_year_map.year,
|
||||
rr = segment_year_map.rr
|
||||
)
|
||||
|
||||
# Add columns for each quantity
|
||||
for q in quantities
|
||||
output[!, Symbol(q)] = zeros(Float64, n_rows)
|
||||
output[!, Symbol("$(q)_se")] = zeros(Float64, n_rows)
|
||||
output[!, Symbol("$(q)_q025")] = zeros(Float64, n_rows)
|
||||
output[!, Symbol("$(q)_q975")] = zeros(Float64, n_rows)
|
||||
end
|
||||
|
||||
println("Extracting estimates for $(n_rows) segment-year positions...")
|
||||
|
||||
# Progress tracking
|
||||
prog_interval = max(1, n_rows ÷ 20)
|
||||
|
||||
for (i, row) in enumerate(eachrow(segment_year_map))
|
||||
r = row.rr
|
||||
|
||||
# Progress
|
||||
if i % prog_interval == 0 || i == n_rows
|
||||
pct = round(100 * i / n_rows, digits=1)
|
||||
print("\r Progress: $pct% ($i / $n_rows)")
|
||||
end
|
||||
|
||||
for q in quantities
|
||||
col_name = Symbol("$q.$r")
|
||||
|
||||
if !hasproperty(chains, col_name)
|
||||
@warn "Column $col_name not found (rr=$r)" maxlog=5
|
||||
continue
|
||||
end
|
||||
|
||||
samples = chains[!, col_name]
|
||||
|
||||
# Compute summary statistics
|
||||
output[i, Symbol(q)] = mean(samples)
|
||||
output[i, Symbol("$(q)_se")] = std(samples)
|
||||
output[i, Symbol("$(q)_q025")] = quantile(samples, 0.025)
|
||||
output[i, Symbol("$(q)_q975")] = quantile(samples, 0.975)
|
||||
end
|
||||
end
|
||||
println() # Newline after progress
|
||||
|
||||
# Remove the rr column from final output (internal only)
|
||||
select!(output, Not(:rr))
|
||||
|
||||
return output
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 5: Validation
|
||||
# =============================================================================
|
||||
|
||||
function validate_output(output::DataFrame, segment_info::Union{DataFrame, Nothing})
|
||||
println("\n" * "="^60)
|
||||
println("VALIDATION CHECKS")
|
||||
println("="^60)
|
||||
|
||||
all_passed = true
|
||||
|
||||
# Detect which columns are present (2D vs 4D model)
|
||||
has_4d = hasproperty(output, :pro_market)
|
||||
|
||||
# Check 1: Range check - all estimates should be in [0, 1]
|
||||
println("\n1. Range check (all values in [0, 1]):")
|
||||
|
||||
if has_4d
|
||||
check_cols = [:pro_market, :pro_welfare, :cosmopolitan, :traditional, :economic_lr, :galtan]
|
||||
else
|
||||
check_cols = [:economic_lr, :galtan]
|
||||
end
|
||||
|
||||
for col in check_cols
|
||||
if !hasproperty(output, col)
|
||||
continue
|
||||
end
|
||||
vals = output[!, col]
|
||||
min_val, max_val = extrema(vals)
|
||||
in_range = min_val >= 0 && max_val <= 1
|
||||
status = in_range ? "PASS" : "FAIL"
|
||||
println(" $col: [$(@sprintf("%.4f", min_val)), $(@sprintf("%.4f", max_val))] - $status")
|
||||
all_passed = all_passed && in_range
|
||||
end
|
||||
|
||||
# Check 2: Anchor party checks
|
||||
println("\n2. Anchor party checks:")
|
||||
|
||||
# Define anchor parties with expected ranges (for 2D model)
|
||||
# Includes both union IDs (V3) and individual constituent IDs (V4)
|
||||
anchor_parties = [
|
||||
(id=211, name="CDU/CSU", country="DE", econ=(0.50, 0.70), galtan=(0.45, 0.70)),
|
||||
(id=1375, name="CDU", country="DE", econ=(0.50, 0.70), galtan=(0.45, 0.65)),
|
||||
(id=1731, name="CSU", country="DE", econ=(0.50, 0.70), galtan=(0.55, 0.75)),
|
||||
(id=383, name="SPD", country="DE", econ=(0.30, 0.50), galtan=(0.30, 0.55)),
|
||||
(id=1516, name="Labour", country="GB", econ=(0.30, 0.55), galtan=(0.30, 0.55)),
|
||||
(id=1567, name="Conservatives", country="GB", econ=(0.55, 0.80), galtan=(0.50, 0.75)),
|
||||
(id=487, name="SAP", country="SE", econ=(0.30, 0.50), galtan=(0.35, 0.55)),
|
||||
]
|
||||
|
||||
n_checked = 0
|
||||
n_passed = 0
|
||||
|
||||
for anchor in anchor_parties
|
||||
party_rows = filter(r -> r.party_id == anchor.id, output)
|
||||
|
||||
if nrow(party_rows) == 0
|
||||
println(" $(anchor.name) ($(anchor.id)): NOT FOUND")
|
||||
continue
|
||||
end
|
||||
|
||||
# Use most recent 20 years of data as reference period
|
||||
max_year = maximum(party_rows.year)
|
||||
ref_rows = filter(r -> r.year >= max_year - 20, party_rows)
|
||||
if nrow(ref_rows) == 0
|
||||
ref_rows = party_rows
|
||||
end
|
||||
|
||||
n_checked += 1
|
||||
|
||||
mean_econ = mean(ref_rows.economic_lr)
|
||||
mean_galtan = mean(ref_rows.galtan)
|
||||
|
||||
econ_ok = anchor.econ[1] <= mean_econ <= anchor.econ[2]
|
||||
galtan_ok = anchor.galtan[1] <= mean_galtan <= anchor.galtan[2]
|
||||
all_ok = econ_ok && galtan_ok
|
||||
|
||||
if all_ok
|
||||
n_passed += 1
|
||||
end
|
||||
|
||||
status = all_ok ? "PASS" : "WARN"
|
||||
econ_marker = econ_ok ? "" : "*"
|
||||
galtan_marker = galtan_ok ? "" : "*"
|
||||
|
||||
@printf(" %-15s econ=%.2f%s [%.2f-%.2f] galtan=%.2f%s [%.2f-%.2f] %s\n",
|
||||
anchor.name, mean_econ, econ_marker, anchor.econ[1], anchor.econ[2],
|
||||
mean_galtan, galtan_marker, anchor.galtan[1], anchor.galtan[2], status)
|
||||
end
|
||||
|
||||
if n_checked > 0
|
||||
println(" Anchor check: $n_passed/$n_checked within expected ranges")
|
||||
println(" Note: Model integrates text + expert data; deviations from expert-only expectations are normal")
|
||||
end
|
||||
|
||||
# Check 3: Coverage check
|
||||
println("\n3. Coverage check:")
|
||||
println(" Total segment-year positions: $(nrow(output))")
|
||||
println(" Unique parties: $(length(unique(output.party_id)))")
|
||||
blank_country_rows = count(ismissing, output.country)
|
||||
println(" Unique countries: $(length(unique(skipmissing(output.country))))")
|
||||
if blank_country_rows == 0
|
||||
println(" Blank country rows: 0 - PASS")
|
||||
else
|
||||
println(" Blank country rows: $blank_country_rows - FAIL")
|
||||
all_passed = false
|
||||
end
|
||||
println(" Year range: $(minimum(output.year)) - $(maximum(output.year))")
|
||||
|
||||
# V10: Check segment distribution
|
||||
segment_counts = combine(groupby(output, :party_id), nrow => :n_years)
|
||||
parties_multi_segment = filter(r -> r.party_id in
|
||||
[p for p in unique(output.party_id) if length(unique(filter(x -> x.party_id == p, output).segment_num)) > 1],
|
||||
segment_counts)
|
||||
if nrow(parties_multi_segment) > 0
|
||||
println("\n Parties with multiple segments: $(length(unique(parties_multi_segment.party_id)))")
|
||||
end
|
||||
|
||||
# Check 4: No duplicates (party_id, segment_num, year should be unique)
|
||||
println("\n4. Duplicate check:")
|
||||
dup_count = nrow(output) - nrow(unique(select(output, :party_id, :segment_num, :year)))
|
||||
if dup_count == 0
|
||||
println(" No duplicate (party_id, segment_num, year) combinations - PASS")
|
||||
else
|
||||
println(" WARNING: Found $dup_count duplicate combinations!")
|
||||
all_passed = false
|
||||
end
|
||||
|
||||
# Check 5: SE reasonableness
|
||||
println("\n5. Standard error check:")
|
||||
se_cols = has_4d ?
|
||||
[:pro_market_se, :pro_welfare_se, :cosmopolitan_se, :traditional_se] :
|
||||
[:economic_lr_se, :galtan_se]
|
||||
|
||||
for col in se_cols
|
||||
if !hasproperty(output, col)
|
||||
continue
|
||||
end
|
||||
vals = output[!, col]
|
||||
mean_se = mean(vals)
|
||||
max_se = maximum(vals)
|
||||
println(" $col: mean=$(@sprintf("%.4f", mean_se)), max=$(@sprintf("%.4f", max_se))")
|
||||
end
|
||||
|
||||
println("\n" * "-"^60)
|
||||
if all_passed
|
||||
println("All validation checks PASSED")
|
||||
else
|
||||
println("Some validation checks FAILED - please review output carefully")
|
||||
end
|
||||
|
||||
return all_passed
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 6: Save output
|
||||
# =============================================================================
|
||||
|
||||
function save_output(output::DataFrame, metadata::Dict, segment_info::Union{DataFrame, Nothing}, run_dir::String; outdir::String="outputs/estimations/latest")
|
||||
println("\n" * "="^60)
|
||||
println("SAVING OUTPUT")
|
||||
println("="^60)
|
||||
|
||||
timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")
|
||||
mkpath(outdir)
|
||||
|
||||
# Delete previous output files
|
||||
for f in readdir(outdir)
|
||||
if startswith(f, "party_positions_") && (endswith(f, ".csv") || endswith(f, ".txt") || endswith(f, ".tex"))
|
||||
rm(joinpath(outdir, f))
|
||||
println(" Deleted old: $f")
|
||||
end
|
||||
end
|
||||
|
||||
# Save main CSV
|
||||
csv_file = joinpath(outdir, "party_positions_$timestamp.csv")
|
||||
CSV.write(csv_file, output)
|
||||
println("Saved: $csv_file")
|
||||
println(" Rows: $(nrow(output))")
|
||||
println(" Columns: $(ncol(output))")
|
||||
|
||||
# Count parties with multiple segments
|
||||
n_multi_segment = 0
|
||||
if segment_info !== nothing
|
||||
party_segment_counts = combine(groupby(segment_info, :party_id), nrow => :n_segments)
|
||||
n_multi_segment = count(r -> r.n_segments > 1, eachrow(party_segment_counts))
|
||||
end
|
||||
|
||||
# Save metadata
|
||||
meta_file = joinpath(outdir, "party_positions_$(timestamp)_metadata.txt")
|
||||
open(meta_file, "w") do f
|
||||
println(f, "Party Positions Dataset - Metadata")
|
||||
println(f, "="^50)
|
||||
println(f, "")
|
||||
println(f, "Generated: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))")
|
||||
println(f, "Source run: $(basename(run_dir))")
|
||||
println(f, "Model file: $(get(metadata, "model_file", "unknown"))")
|
||||
println(f, "")
|
||||
println(f, "Dataset size:")
|
||||
println(f, " Segment-year observations: $(nrow(output))")
|
||||
println(f, " Unique parties: $(length(unique(output.party_id)))")
|
||||
if n_multi_segment > 0
|
||||
println(f, " Parties with multiple segments: $n_multi_segment")
|
||||
end
|
||||
println(f, " Unique countries: $(length(unique(output.country)))")
|
||||
println(f, " Year range: $(minimum(output.year)) - $(maximum(output.year))")
|
||||
println(f, "")
|
||||
println(f, "Columns:")
|
||||
println(f, " party_id: PartyFacts ID (integer) - individual party (e.g., CDU=1375, CSU=1731)")
|
||||
println(f, " union_party_id: PartyFacts ID of parent union (NA for standalone parties)")
|
||||
println(f, " segment_num: Segment number within party (1, 2, 3...)")
|
||||
println(f, " country: ISO2 country code")
|
||||
println(f, " year: Calendar year")
|
||||
println(f, "")
|
||||
println(f, "Segment-Based Indexing:")
|
||||
println(f, " - Parties are split into segments at gaps > 7 years")
|
||||
println(f, " - Each segment is estimated independently (no continuity across gaps)")
|
||||
println(f, " - Segments with < 3 observations are dropped")
|
||||
println(f, " - segment_num=1 is the main segment; higher numbers indicate gaps in data")
|
||||
println(f, "")
|
||||
|
||||
# Check if this is 2D or 4D output
|
||||
is_2d = !hasproperty(output, :pro_market)
|
||||
|
||||
if is_2d
|
||||
println(f, "Model: 2D Direct Bipolar")
|
||||
println(f, "")
|
||||
println(f, "Bipolar scales (0 = left/GAL, 1 = right/TAN):")
|
||||
println(f, " economic_lr: Economic left-right position (directly estimated)")
|
||||
println(f, " galtan: GAL-TAN cultural position (directly estimated)")
|
||||
# Note: general_lr is computed internally for cross-dimensional anchoring
|
||||
# but not reported as output (the two dimension-specific estimates are preferred)
|
||||
else
|
||||
println(f, "Model: 4D Unipolar")
|
||||
println(f, "")
|
||||
println(f, "Dimensions (0 = low, 1 = high):")
|
||||
println(f, " pro_market: Pro-market economic position")
|
||||
println(f, " pro_welfare: Pro-welfare state position")
|
||||
println(f, " cosmopolitan: Cosmopolitan/GAL cultural position")
|
||||
println(f, " traditional: Traditional/TAN cultural position")
|
||||
println(f, "")
|
||||
println(f, "Derived bipolar scales (0 = left/GAL, 1 = right/TAN):")
|
||||
println(f, " economic_lr: Economic left-right (derived from pro_market - pro_welfare)")
|
||||
println(f, " galtan: GAL-TAN (derived from traditional - cosmopolitan)")
|
||||
end
|
||||
println(f, "")
|
||||
println(f, "Uncertainty columns:")
|
||||
println(f, " *_se: Standard error (posterior SD)")
|
||||
println(f, " *_q025: 2.5th percentile (lower 95% CI)")
|
||||
println(f, " *_q975: 97.5th percentile (upper 95% CI)")
|
||||
println(f, "")
|
||||
println(f, "Model convergence:")
|
||||
println(f, " Mean R-hat: $(get(metadata, "mean_rhat", "N/A"))")
|
||||
println(f, " Max R-hat: $(get(metadata, "max_rhat", "N/A"))")
|
||||
println(f, " Mean ESS: $(get(metadata, "mean_ess", "N/A"))")
|
||||
println(f, " Min ESS: $(get(metadata, "min_ess", "N/A"))")
|
||||
end
|
||||
println("Saved: $meta_file")
|
||||
|
||||
|
||||
# =============================================================================
|
||||
# STEP 5b: Verify no union/alliance IDs in output
|
||||
# =============================================================================
|
||||
|
||||
function verify_no_unions_in_output(output::DataFrame)
|
||||
println("\n" * "="^60)
|
||||
println("UNION ID VERIFICATION")
|
||||
println("="^60)
|
||||
|
||||
union_mapping_file = joinpath("data", "union_mapping.csv")
|
||||
if !isfile(union_mapping_file)
|
||||
println(" No union_mapping.csv found — skipping verification")
|
||||
return
|
||||
end
|
||||
|
||||
union_df = CSV.read(union_mapping_file, DataFrame)
|
||||
union_pf_ids = Set(union_df.manifesto_pf_id)
|
||||
output_pf_ids = Set(output.party_id)
|
||||
|
||||
violations = intersect(union_pf_ids, output_pf_ids)
|
||||
|
||||
if isempty(violations)
|
||||
println(" PASS: No union/alliance PF IDs found in output")
|
||||
println(" Checked $(length(union_pf_ids)) union IDs against $(length(output_pf_ids)) output parties")
|
||||
else
|
||||
println(" WARNING: $(length(violations)) union PF IDs found in output")
|
||||
println(" (This is expected if union_mapping.csv was updated after the model run)")
|
||||
for v in sort(collect(violations))
|
||||
n_rows = count(r -> r.party_id == v, eachrow(output))
|
||||
println(" PF $v: $n_rows rows")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# MAIN
|
||||
# =============================================================================
|
||||
|
||||
function main()
|
||||
println("="^60)
|
||||
println("POST-ESTIMATION: 4D Latent Trait Model (V10)")
|
||||
println("="^60)
|
||||
println("Started: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))")
|
||||
|
||||
# Step 0: Find run directory (CLI --run-dir or auto-detect latest)
|
||||
run_dir = nothing
|
||||
output_dir = nothing
|
||||
for (i, arg) in enumerate(ARGS)
|
||||
if arg == "--run-dir" && i < length(ARGS)
|
||||
run_dir = ARGS[i + 1]
|
||||
elseif startswith(arg, "--run-dir=")
|
||||
run_dir = split(arg, "=", limit=2)[2]
|
||||
elseif arg == "--output-dir" && i < length(ARGS)
|
||||
output_dir = ARGS[i + 1]
|
||||
elseif startswith(arg, "--output-dir=")
|
||||
output_dir = split(arg, "=", limit=2)[2]
|
||||
end
|
||||
end
|
||||
if run_dir === nothing
|
||||
run_dir = find_latest_run()
|
||||
else
|
||||
println("Using specified run directory: $run_dir")
|
||||
end
|
||||
|
||||
# Step 1: Load run data
|
||||
data = load_run_data(run_dir)
|
||||
|
||||
# Step 2: Build segment-year mapping
|
||||
year0 = data.metadata["year0"]
|
||||
segment_year_map, R, segment_info = build_segment_year_map(
|
||||
data.text_data, data.expert_dim, data.expert_lr,
|
||||
data.segment_info, data.segment_year_map, data.run_dir, year0
|
||||
)
|
||||
|
||||
# Step 3: Load chains
|
||||
chains = load_chains(data.chain_files)
|
||||
|
||||
# Step 4: Extract estimates
|
||||
output = extract_estimates(chains, segment_year_map, R)
|
||||
|
||||
# Step 5: Validate
|
||||
validate_output(output, segment_info)
|
||||
|
||||
# Step 5b: Verify no union/alliance IDs in output
|
||||
verify_no_unions_in_output(output)
|
||||
|
||||
# Step 6: Save output
|
||||
effective_output_dir = output_dir !== nothing ? output_dir : "outputs/estimations/latest"
|
||||
csv_file, meta_file = save_output(output, data.metadata, segment_info, run_dir; outdir=effective_output_dir)
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("COMPLETE")
|
||||
println("="^60)
|
||||
println("Output files:")
|
||||
println(" $csv_file")
|
||||
println(" $meta_file")
|
||||
println("\nFinished: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))")
|
||||
|
||||
return output
|
||||
end
|
||||
|
||||
# Run if executed directly
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
main()
|
||||
end
|
||||
@@ -0,0 +1,332 @@
|
||||
#!/usr/bin/env julia
|
||||
#############################################################################
|
||||
## 00_validation.jl
|
||||
## Pre-flight validation checks for Stan data and initialization
|
||||
## Prevents cryptic Stan errors by catching issues early
|
||||
#############################################################################
|
||||
|
||||
using Statistics
|
||||
|
||||
"""
|
||||
Validate Stan data dictionary before passing to Stan.
|
||||
Catches common issues that cause Stan to crash with cryptic errors.
|
||||
"""
|
||||
function validate_stan_data(dat::Dict; verbose=true)
|
||||
verbose && println("\n" * "="^70)
|
||||
verbose && println("VALIDATING STAN DATA")
|
||||
verbose && println("="^70)
|
||||
|
||||
errors = String[]
|
||||
warnings = String[]
|
||||
|
||||
# Check for NaN/Inf in all numeric data
|
||||
for (key, value) in dat
|
||||
if isa(value, AbstractArray) && eltype(value) <: Number
|
||||
if any(isnan, value)
|
||||
push!(errors, "Data '$key' contains NaN values")
|
||||
end
|
||||
if any(isinf, value)
|
||||
push!(errors, "Data '$key' contains Inf values")
|
||||
end
|
||||
elseif isa(value, Number)
|
||||
if isnan(value)
|
||||
push!(errors, "Data '$key' is NaN")
|
||||
end
|
||||
if isinf(value)
|
||||
push!(errors, "Data '$key' is Inf")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Validate expert data is in open interval (0, 1)
|
||||
if haskey(dat, "val_dim")
|
||||
val_dim = dat["val_dim"]
|
||||
if any(x -> x <= 0 || x >= 1, val_dim)
|
||||
n_boundary = count(x -> x <= 0 || x >= 1, val_dim)
|
||||
push!(errors, "Expert dimension data has $n_boundary values at boundaries (must be in (0,1))")
|
||||
if verbose
|
||||
println(" Dimension-specific expert data range: [$(minimum(val_dim)), $(maximum(val_dim))]")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if haskey(dat, "val_lr")
|
||||
val_lr = dat["val_lr"]
|
||||
if any(x -> x <= 0 || x >= 1, val_lr)
|
||||
n_boundary = count(x -> x <= 0 || x >= 1, val_lr)
|
||||
push!(errors, "Expert L-R data has $n_boundary values at boundaries (must be in (0,1))")
|
||||
if verbose
|
||||
println(" L-R expert data range: [$(minimum(val_lr)), $(maximum(val_lr))]")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Validate manifesto data
|
||||
if haskey(dat, "positive") && haskey(dat, "sample")
|
||||
positive = dat["positive"]
|
||||
sample = dat["sample"]
|
||||
|
||||
if any(positive .> sample)
|
||||
push!(errors, "Manifesto: positive counts exceed sample sizes")
|
||||
end
|
||||
|
||||
if any(positive .< 0)
|
||||
push!(errors, "Manifesto: negative positive counts found")
|
||||
end
|
||||
|
||||
if any(sample .< 0)
|
||||
push!(errors, "Manifesto: negative sample sizes found")
|
||||
end
|
||||
end
|
||||
|
||||
# Validate indices are within bounds
|
||||
# V10: Check segment indices (ss_man) if present, otherwise party indices (jj_man)
|
||||
if haskey(dat, "S") && haskey(dat, "ss_man")
|
||||
S = dat["S"]
|
||||
ss_man = dat["ss_man"]
|
||||
if any(ss_man .< 1) || any(ss_man .> S)
|
||||
push!(errors, "Manifesto segment indices out of bounds [1, $S]")
|
||||
end
|
||||
elseif haskey(dat, "J") && haskey(dat, "jj_man")
|
||||
J = dat["J"]
|
||||
jj_man = dat["jj_man"]
|
||||
if any(jj_man .< 1) || any(jj_man .> J)
|
||||
push!(errors, "Manifesto party indices out of bounds [1, $J]")
|
||||
end
|
||||
end
|
||||
|
||||
if haskey(dat, "R") && haskey(dat, "rr_man")
|
||||
R = dat["R"]
|
||||
rr_man = dat["rr_man"]
|
||||
if any(rr_man .< 1) || any(rr_man .> R)
|
||||
push!(errors, "Manifesto party-year indices out of bounds [1, $R]")
|
||||
end
|
||||
end
|
||||
|
||||
# V4: Validate constituent arrays
|
||||
if haskey(dat, "const_rr_man") && haskey(dat, "R")
|
||||
R = dat["R"]
|
||||
const_rr = dat["const_rr_man"]
|
||||
if any(const_rr .< 1) || any(const_rr .> R)
|
||||
push!(errors, "const_rr_man out of bounds [1, $R]")
|
||||
end
|
||||
# Verify offsets are valid
|
||||
if haskey(dat, "const_offset_man") && haskey(dat, "n_const_man")
|
||||
offsets = dat["const_offset_man"]
|
||||
n_consts = dat["n_const_man"]
|
||||
total = dat["N_const_man_total"]
|
||||
for i in eachindex(offsets)
|
||||
if offsets[i] + n_consts[i] - 1 > total
|
||||
push!(errors, "const_offset_man[$i] + n_const_man[$i] exceeds N_const_man_total")
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Print summary
|
||||
if verbose
|
||||
println("\nDATA SUMMARY:")
|
||||
# V10: Show segments if present
|
||||
if haskey(dat, "S")
|
||||
println(" Segments (S): $(dat["S"])")
|
||||
println(" Parties with segments (J): $(get(dat, "J", "N/A"))")
|
||||
else
|
||||
println(" Parties (J): $(get(dat, "J", "N/A"))")
|
||||
end
|
||||
println(" Countries (P): $(get(dat, "P", "N/A"))")
|
||||
println(" Segment-years (R): $(get(dat, "R", "N/A"))")
|
||||
println(" Years (T_year): $(get(dat, "T_year", "N/A"))")
|
||||
println(" Manifesto obs: $(get(dat, "N_man", "N/A"))")
|
||||
println(" Expert dim obs: $(get(dat, "N_exp_dim", "N/A"))")
|
||||
println(" Expert L-R obs: $(get(dat, "N_exp_lr", "N/A"))")
|
||||
|
||||
if haskey(dat, "mn_resp_log_man")
|
||||
println("\nPRIOR MEANS:")
|
||||
println(" Manifesto: $(round(dat["mn_resp_log_man"], digits=3))")
|
||||
println(" Expert dim: $(round(dat["mn_resp_log_exp_dim"], digits=3))")
|
||||
println(" Expert L-R: $(round(dat["mn_resp_log_exp_lr"], digits=3))")
|
||||
end
|
||||
end
|
||||
|
||||
# Report results
|
||||
if !isempty(errors)
|
||||
println("\n❌ VALIDATION FAILED - $(length(errors)) ERROR(S):")
|
||||
for (i, err) in enumerate(errors)
|
||||
println(" $i. $err")
|
||||
end
|
||||
return false
|
||||
end
|
||||
|
||||
if !isempty(warnings)
|
||||
println("\n⚠️ $(length(warnings)) WARNING(S):")
|
||||
for (i, warn) in enumerate(warnings)
|
||||
println(" $i. $warn")
|
||||
end
|
||||
end
|
||||
|
||||
if verbose
|
||||
println("\n✓ DATA VALIDATION PASSED")
|
||||
println("="^70)
|
||||
end
|
||||
|
||||
return true
|
||||
end
|
||||
|
||||
"""
|
||||
Validate initialization values before passing to Stan.
|
||||
Checks for common issues that cause immediate Stan crashes.
|
||||
"""
|
||||
function validate_init_values(init_dict::Dict; verbose=true)
|
||||
verbose && println("\n" * "="^70)
|
||||
verbose && println("VALIDATING INITIALIZATION VALUES")
|
||||
verbose && println("="^70)
|
||||
|
||||
errors = String[]
|
||||
warnings = String[]
|
||||
|
||||
for (key, value) in init_dict
|
||||
# Check for NaN/Inf
|
||||
if isa(value, AbstractArray) && eltype(value) <: Number
|
||||
if any(isnan, value)
|
||||
push!(errors, "Init '$key' contains NaN")
|
||||
end
|
||||
if any(isinf, value)
|
||||
push!(errors, "Init '$key' contains Inf")
|
||||
end
|
||||
|
||||
if verbose && length(value) > 0
|
||||
val_array = vec(value)
|
||||
println(" $key: range [$(round(minimum(val_array), digits=3)), $(round(maximum(val_array), digits=3))]")
|
||||
end
|
||||
elseif isa(value, Number)
|
||||
if isnan(value)
|
||||
push!(errors, "Init '$key' is NaN")
|
||||
end
|
||||
if isinf(value)
|
||||
push!(errors, "Init '$key' is Inf")
|
||||
end
|
||||
|
||||
if verbose
|
||||
println(" $key: $(round(value, digits=3))")
|
||||
end
|
||||
end
|
||||
|
||||
# Check positive constraints (common Stan constraints)
|
||||
# Exception: *_raw parameters are non-centered and can be any real
|
||||
if (contains(string(key), "sigma") || contains(string(key), "tau") || contains(string(key), "phi")) &&
|
||||
!endswith(string(key), "_raw")
|
||||
if isa(value, Number) && value <= 0
|
||||
push!(errors, "Init '$key' = $value violates constraint > 0")
|
||||
elseif isa(value, AbstractArray) && any(value .<= 0)
|
||||
push!(errors, "Init '$key' has values ≤ 0 (violates constraint > 0)")
|
||||
end
|
||||
end
|
||||
|
||||
# Check Cholesky factors are valid
|
||||
if contains(string(key), "L_Omega")
|
||||
if isa(value, AbstractMatrix)
|
||||
# Check it's lower triangular with positive diagonal
|
||||
n = size(value, 1)
|
||||
if size(value, 2) != n
|
||||
push!(errors, "Init '$key' is not square")
|
||||
end
|
||||
for i in 1:n
|
||||
if value[i, i] <= 0
|
||||
push!(errors, "Init '$key' has non-positive diagonal at position $i")
|
||||
end
|
||||
for j in (i+1):n
|
||||
if abs(value[i, j]) > 1e-10
|
||||
push!(warnings, "Init '$key' is not lower triangular")
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Check slope parameters for positive constraint (V2/V3 feature)
|
||||
if key == "Gamma_man_slope_raw"
|
||||
if isa(value, AbstractArray) && any(value .< 0)
|
||||
push!(errors, "Init 'Gamma_man_slope_raw' has negative values (must be ≥ 0)")
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Report results
|
||||
if !isempty(errors)
|
||||
println("\n❌ INIT VALIDATION FAILED - $(length(errors)) ERROR(S):")
|
||||
for (i, err) in enumerate(errors)
|
||||
println(" $i. $err")
|
||||
end
|
||||
return false
|
||||
end
|
||||
|
||||
if !isempty(warnings)
|
||||
println("\n⚠️ $(length(warnings)) WARNING(S):")
|
||||
for (i, warn) in enumerate(warnings)
|
||||
println(" $i. $warn")
|
||||
end
|
||||
end
|
||||
|
||||
if verbose
|
||||
println("\n✓ INIT VALIDATION PASSED")
|
||||
println("="^70)
|
||||
end
|
||||
|
||||
return true
|
||||
end
|
||||
|
||||
"""
|
||||
Estimate memory requirements for model
|
||||
"""
|
||||
function estimate_memory_requirements(dat::Dict; verbose=true, num_chains::Int=4, num_samples::Int=1000, num_threads_per_chain::Int=1)
|
||||
if !verbose
|
||||
return
|
||||
end
|
||||
|
||||
println("\n" * "="^70)
|
||||
println("MEMORY ESTIMATE")
|
||||
println("="^70)
|
||||
|
||||
R = get(dat, "R", 0)
|
||||
J = get(dat, "J", 0)
|
||||
K_man = get(dat, "K_man", 0)
|
||||
K_exp_dim = get(dat, "K_exp_dim", 0)
|
||||
K_exp_lr = get(dat, "K_exp_lr", 0)
|
||||
N_man = get(dat, "N_man", 0)
|
||||
N_ciy = get(dat, "N_ciy", 0)
|
||||
T_year = get(dat, "T_year", 0)
|
||||
|
||||
# Rough parameter count
|
||||
theta_params = 4 * R
|
||||
item_params = 4 * K_man * 2 + K_exp_dim * 3 + K_exp_lr * 2
|
||||
strategic_params = get(dat, "P", 0) * get(dat, "K_man", 0) # Country-item intercepts
|
||||
other_params = 4 * J + T_year + J + N_ciy + 50
|
||||
|
||||
total_params = theta_params + item_params + strategic_params + other_params
|
||||
|
||||
# Memory estimate (very rough)
|
||||
# Each parameter: ~8 bytes (float64) × samples × chains
|
||||
total_draws_per_param = num_samples * num_chains
|
||||
bytes_per_param = 8 * total_draws_per_param
|
||||
total_mb = (total_params * bytes_per_param) / (1024 * 1024)
|
||||
|
||||
println(" Configuration:")
|
||||
println(" Chains: $num_chains")
|
||||
println(" Samples per chain: $num_samples")
|
||||
println(" Threads per chain: $num_threads_per_chain")
|
||||
println(" Total parallel workers: $(num_chains * num_threads_per_chain)")
|
||||
|
||||
println(" Total parameters: ~$(total_params)")
|
||||
println(" Estimated memory (samples only): ~$(round(total_mb, digits=0)) MB")
|
||||
thread_scaling = max(1, num_threads_per_chain)
|
||||
println(" With thread overhead (×$(thread_scaling)): ~$(round(total_mb * thread_scaling, digits=0)) MB")
|
||||
println(" With safety margin (×3): ~$(round(3 * total_mb * thread_scaling, digits=0)) MB")
|
||||
|
||||
if total_mb * thread_scaling * 3 > 8000
|
||||
println("\n⚠️ WARNING: Model may require > 8GB RAM")
|
||||
end
|
||||
|
||||
println("="^70)
|
||||
end
|
||||
@@ -0,0 +1,277 @@
|
||||
#!/usr/bin/env julia
|
||||
#############################################################################
|
||||
## 02_data_loading.jl
|
||||
## Load and preprocess data for latent trait model
|
||||
## Loads three datasets: text_data (manifesto + PolDem), expert dimension-specific, expert general L-R
|
||||
##
|
||||
## Supports both:
|
||||
## - 4D model (V10): type_high/type_low columns for bipolar bridges
|
||||
## - 2D model (V1): dim_idx + direction columns for direct estimation
|
||||
#############################################################################
|
||||
|
||||
using DataFrames, CSV, CategoricalArrays, Statistics
|
||||
|
||||
#############################################################################
|
||||
## UNION MAPPING: Individual party estimates via mean-constituent model
|
||||
## Loads data/union_mapping.csv and builds lookup structures
|
||||
#############################################################################
|
||||
|
||||
"""
|
||||
load_union_mapping(project_root::String)
|
||||
|
||||
Load union_mapping.csv and build lookup dictionaries.
|
||||
Returns (union_to_constituents, constituent_to_union) dicts.
|
||||
If file is missing or empty, returns empty dicts (backwards compatible).
|
||||
"""
|
||||
function load_union_mapping(project_root::String=".")
|
||||
mapping_file = joinpath(project_root, "data", "union_mapping.csv")
|
||||
|
||||
union_to_constituents = Dict{Int, Vector{Int}}()
|
||||
constituent_to_union = Dict{Int, Int}()
|
||||
|
||||
if !isfile(mapping_file)
|
||||
println(" No union_mapping.csv found - running without union decomposition")
|
||||
return union_to_constituents, constituent_to_union
|
||||
end
|
||||
|
||||
df = CSV.read(mapping_file, DataFrame)
|
||||
if nrow(df) == 0
|
||||
println(" union_mapping.csv is empty - running without union decomposition")
|
||||
return union_to_constituents, constituent_to_union
|
||||
end
|
||||
|
||||
for row in eachrow(df)
|
||||
union_id = row.manifesto_pf_id
|
||||
expert_id = row.expert_pf_id
|
||||
|
||||
if !haskey(union_to_constituents, union_id)
|
||||
union_to_constituents[union_id] = Int[]
|
||||
end
|
||||
if !(expert_id in union_to_constituents[union_id])
|
||||
push!(union_to_constituents[union_id], expert_id)
|
||||
end
|
||||
constituent_to_union[expert_id] = union_id
|
||||
end
|
||||
|
||||
println(" Union mapping loaded: $(length(union_to_constituents)) unions, $(length(constituent_to_union)) constituents")
|
||||
return union_to_constituents, constituent_to_union
|
||||
end
|
||||
|
||||
#############################################################################
|
||||
## SEGMENT-BASED INDEXING CONFIGURATION
|
||||
## Split parties at gaps > MAX_GAP years to avoid flat posteriors
|
||||
#############################################################################
|
||||
const MAX_GAP = 7 # Maximum years between observations within a segment
|
||||
const MIN_OBS = 2 # Minimum observations per segment (drop segments with fewer)
|
||||
|
||||
#############################################################################
|
||||
## 2D MODEL MAPPING CONFIGURATION
|
||||
## Maps type_high/type_low pairs to dim_idx + direction
|
||||
#############################################################################
|
||||
const TYPE_TO_DIM_DIRECTION = Dict(
|
||||
# Economic dimension: pro_market = right (+1), pro_welfare = left (-1)
|
||||
("pro_market", "pro_welfare") => (dim_idx=1, direction=1), # Right
|
||||
("pro_welfare", "pro_market") => (dim_idx=1, direction=-1), # Left
|
||||
|
||||
# Cultural dimension: traditional = TAN (+1), cosmopolitan = GAL (-1)
|
||||
("traditional", "cosmopolitan") => (dim_idx=2, direction=1), # TAN
|
||||
("cosmopolitan", "traditional") => (dim_idx=2, direction=-1) # GAL
|
||||
)
|
||||
|
||||
# Expert dimension mapping (lrecon -> economic, galtan/cultural -> galtan)
|
||||
const EXPERT_VAR_TO_DIM = Dict(
|
||||
"lrecon_ches" => 1,
|
||||
"lrecon_vparty" => 1,
|
||||
"welf_vparty" => 1,
|
||||
"lrecon_gps" => 1,
|
||||
"lrecon_poppa" => 1,
|
||||
"galtan_ches" => 2,
|
||||
"libcon_gps" => 2,
|
||||
"immig_vparty" => 2,
|
||||
"lgbt_vparty" => 2,
|
||||
"culsup_vparty" => 2,
|
||||
"relig_vparty" => 2,
|
||||
"gender_vparty" => 2
|
||||
)
|
||||
|
||||
function load_and_preprocess_4dim_data(start_year=1950; data_dir::String=".")
|
||||
println("Loading 4D latent trait data files...")
|
||||
println("Start year filter: $start_year")
|
||||
data_dir != "." && println("Data directory: $data_dir")
|
||||
|
||||
# Load union mapping (check data_dir first, fall back to project root)
|
||||
println("\nLoading union mapping...")
|
||||
union_mapping_dir = isfile(joinpath(data_dir, "data", "union_mapping.csv")) ? data_dir : "."
|
||||
union_to_constituents, constituent_to_union = load_union_mapping(union_mapping_dir)
|
||||
|
||||
# Load the three datasets
|
||||
text_data_raw = CSV.read(joinpath(data_dir, "text_data.csv"), DataFrame)
|
||||
expert_raw = CSV.read(joinpath(data_dir, "expert.csv"), DataFrame)
|
||||
lr_data_raw = CSV.read(joinpath(data_dir, "lr_data.csv"), DataFrame)
|
||||
|
||||
# Filter to start year BEFORE calculating year0
|
||||
text_data_raw = text_data_raw[text_data_raw.year .>= start_year, :]
|
||||
expert_raw = expert_raw[expert_raw.year .>= start_year, :]
|
||||
lr_data_raw = lr_data_raw[lr_data_raw.year .>= start_year, :]
|
||||
|
||||
println("Data filtered to $start_year onwards:")
|
||||
println(" Text data (manifesto + PolDem): $(nrow(text_data_raw)) observations")
|
||||
println(" Expert: $(nrow(expert_raw)) observations")
|
||||
println(" L-R data: $(nrow(lr_data_raw)) observations")
|
||||
|
||||
# Define base year for relative time indexing
|
||||
year0 = Int(minimum([minimum(text_data_raw.year), minimum(expert_raw.year), minimum(lr_data_raw.year)])) - 1
|
||||
println("Base year set to: $year0")
|
||||
|
||||
# Create type mapping for the four dimensions
|
||||
type_map = Dict(
|
||||
"pro_market" => 1,
|
||||
"pro_welfare" => 2,
|
||||
"cosmopolitan" => 3,
|
||||
"traditional" => 4
|
||||
)
|
||||
|
||||
println("Type mapping: pro_market=1, pro_welfare=2, cosmopolitan=3, traditional=4")
|
||||
|
||||
# Process text data (manifesto + PolDem media)
|
||||
text_data = copy(text_data_raw)
|
||||
text_data = text_data[text_data.year .> year0, :]
|
||||
|
||||
# Add type indices for text items (V4/V10: bipolar bridge structure)
|
||||
if !("type_high" in names(text_data)) || !("type_low" in names(text_data))
|
||||
error("Text data must contain 'type_high' and 'type_low' columns with values: pro_market, pro_welfare, cosmopolitan, traditional")
|
||||
end
|
||||
|
||||
text_data.type_high_idx = [type_map[t] for t in text_data.type_high]
|
||||
text_data.type_low_idx = [type_map[t] for t in text_data.type_low]
|
||||
|
||||
# V1 (2D model): Add dim_idx and direction columns
|
||||
# Maps type_high/type_low to single dimension + direction
|
||||
dim_idx_man = Int[]
|
||||
direction_man = Int[]
|
||||
for row in eachrow(text_data)
|
||||
key = (row.type_high, row.type_low)
|
||||
if haskey(TYPE_TO_DIM_DIRECTION, key)
|
||||
mapping = TYPE_TO_DIM_DIRECTION[key]
|
||||
push!(dim_idx_man, mapping.dim_idx)
|
||||
push!(direction_man, mapping.direction)
|
||||
else
|
||||
# Unknown mapping - this should not happen with valid data
|
||||
error("Unknown type_high/type_low pair: $(row.type_high) / $(row.type_low)")
|
||||
end
|
||||
end
|
||||
text_data.dim_idx_man = dim_idx_man
|
||||
text_data.direction_man = direction_man
|
||||
|
||||
# Standard processing
|
||||
text_data.country = categorical(text_data.country)
|
||||
text_data.party = categorical(text_data.party)
|
||||
text_data.var = categorical(text_data.var)
|
||||
text_data.Year = Int.(text_data.year) .- year0
|
||||
sort!(text_data, [:country, :party, :year, :var])
|
||||
println("Text data processed: $(nrow(text_data)) observations with bipolar bridge structure")
|
||||
|
||||
# Process expert dimension-specific data (bipolar items like lrecon_ches, galtan_ches)
|
||||
expert_dim_vars = ["lrecon_ches", "galtan_ches", "lrecon_vparty", "welf_vparty",
|
||||
"lrecon_gps", "libcon_gps", "lrecon_poppa",
|
||||
"immig_vparty", "lgbt_vparty", "culsup_vparty", "relig_vparty", "gender_vparty"]
|
||||
expert_dim = expert_raw[in.(expert_raw.var, Ref(expert_dim_vars)), :]
|
||||
expert_dim = expert_dim[(expert_dim.year .> year0) .& (expert_dim.val .>= 0) .& (expert_dim.val .<= 1), :]
|
||||
|
||||
# V5: Load integer observations, scale sizes, and expert counts for beta-binomial likelihood
|
||||
expert_dim.val_int = Int.(expert_dim.val_int)
|
||||
expert_dim.n_scale = Int.(expert_dim.n_scale)
|
||||
expert_dim.n_experts = Int.(expert_dim.n_experts)
|
||||
|
||||
# Add type mappings for dimension-specific expert data
|
||||
if !("type_low" in names(expert_dim)) || !("type_high" in names(expert_dim))
|
||||
error("Expert data must contain 'type_low' and 'type_high' columns")
|
||||
end
|
||||
|
||||
expert_dim.type_high_idx = [type_map[t] for t in expert_dim.type_high]
|
||||
expert_dim.type_low_idx = [type_map[t] for t in expert_dim.type_low]
|
||||
|
||||
# V1 (2D model): Add dim_idx for expert dimension data
|
||||
dim_idx_exp = Int[]
|
||||
for row in eachrow(expert_dim)
|
||||
var_name = string(row.var)
|
||||
if haskey(EXPERT_VAR_TO_DIM, var_name)
|
||||
push!(dim_idx_exp, EXPERT_VAR_TO_DIM[var_name])
|
||||
else
|
||||
# Fallback: infer from type_high/type_low
|
||||
key = (row.type_high, row.type_low)
|
||||
if haskey(TYPE_TO_DIM_DIRECTION, key)
|
||||
push!(dim_idx_exp, TYPE_TO_DIM_DIRECTION[key].dim_idx)
|
||||
else
|
||||
error("Unknown expert variable: $var_name with type pair $(row.type_high) / $(row.type_low)")
|
||||
end
|
||||
end
|
||||
end
|
||||
expert_dim.dim_idx_exp = dim_idx_exp
|
||||
|
||||
expert_dim.country = categorical(expert_dim.country)
|
||||
expert_dim.party = categorical(expert_dim.party)
|
||||
expert_dim.var = categorical(expert_dim.var)
|
||||
expert_dim.Year = Int.(expert_dim.year) .- year0
|
||||
sort!(expert_dim, [:country, :party, :year, :var])
|
||||
println("Expert dimension-specific data processed: $(nrow(expert_dim)) observations")
|
||||
|
||||
# Process expert general L-R data (cross-dimensional anchoring)
|
||||
lr_vars = ["lr_ches", "lr_poppa", "lr_morgan"] # General left-right items
|
||||
expert_lr = lr_data_raw[in.(lr_data_raw.var, Ref(lr_vars)), :]
|
||||
expert_lr = expert_lr[(expert_lr.year .> year0) .& (expert_lr.val .>= 0) .& (expert_lr.val .<= 1), :]
|
||||
|
||||
# V5: Load integer observations, scale sizes, and expert counts for beta-binomial likelihood
|
||||
expert_lr.val_int = Int.(expert_lr.val_int)
|
||||
expert_lr.n_scale = Int.(expert_lr.n_scale)
|
||||
expert_lr.n_experts = Int.(expert_lr.n_experts)
|
||||
|
||||
expert_lr.country = categorical(expert_lr.country)
|
||||
expert_lr.party = categorical(expert_lr.party)
|
||||
expert_lr.var = categorical(expert_lr.var)
|
||||
expert_lr.Year = Int.(expert_lr.year) .- year0
|
||||
sort!(expert_lr, [:country, :party, :year, :var])
|
||||
println("Expert general L-R data processed: $(nrow(expert_lr)) observations")
|
||||
|
||||
# Validate data integrity
|
||||
println("\nData validation:")
|
||||
|
||||
# Check text data dimension pair distribution (V4: bipolar bridges)
|
||||
type_pair_counts = combine(groupby(text_data, [:type_high, :type_low]), nrow => :count)
|
||||
for row in eachrow(type_pair_counts)
|
||||
println(" $(row.type_high) ↔ $(row.type_low): $(row.count) text data observations")
|
||||
end
|
||||
|
||||
# Check expert dimension-specific type pairs
|
||||
type_pair_counts = combine(groupby(expert_dim, [:type_high, :type_low]), nrow => :count)
|
||||
for row in eachrow(type_pair_counts)
|
||||
println(" $(row.type_high) - $(row.type_low): $(row.count) expert dimension-specific observations")
|
||||
end
|
||||
|
||||
# Check general L-R items
|
||||
lr_var_counts = combine(groupby(expert_lr, :var), nrow => :count)
|
||||
for row in eachrow(lr_var_counts)
|
||||
println(" $(row.var): $(row.count) general L-R observations")
|
||||
end
|
||||
|
||||
# Check overlapping parties across datasets
|
||||
text_data_parties = Set(text_data.party)
|
||||
expert_dim_parties = Set(expert_dim.party)
|
||||
expert_lr_parties = Set(expert_lr.party)
|
||||
|
||||
all_parties = union(text_data_parties, expert_dim_parties, expert_lr_parties)
|
||||
println("\nParty coverage:")
|
||||
println(" Total unique parties: $(length(all_parties))")
|
||||
println(" In text data: $(length(text_data_parties))")
|
||||
println(" In expert dimension-specific: $(length(expert_dim_parties))")
|
||||
println(" In expert general L-R: $(length(expert_lr_parties))")
|
||||
println(" In all three datasets: $(length(intersect(text_data_parties, expert_dim_parties, expert_lr_parties)))")
|
||||
|
||||
return text_data, expert_dim, expert_lr, year0, union_to_constituents, constituent_to_union
|
||||
end
|
||||
|
||||
# Execute if run directly
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
text_data, expert_dim, expert_lr, year0, u2c, c2u = load_and_preprocess_4dim_data()
|
||||
println("4D data loading test completed successfully")
|
||||
end
|
||||
@@ -0,0 +1,947 @@
|
||||
#!/usr/bin/env julia
|
||||
#############################################################################
|
||||
## 03_data_preparation_4dim.jl
|
||||
## V10: Segment-based indexing to fix long gap interpolation issues
|
||||
##
|
||||
## Key change: Split parties at gaps > MAX_GAP years into independent segments.
|
||||
## Each segment has its own random walk (restarts at segment start).
|
||||
## Segments with < MIN_OBS observations are dropped.
|
||||
#############################################################################
|
||||
|
||||
using DataFrames, CSV, CategoricalArrays, Statistics, StatsFuns
|
||||
|
||||
# Import configuration from data loading module
|
||||
include("02_data_loading.jl")
|
||||
|
||||
"""
|
||||
split_party_years_into_segments(years::Vector{Int}, max_gap::Int)
|
||||
|
||||
Split a party's observation years into segments based on gaps.
|
||||
Returns a vector of vectors, where each inner vector contains consecutive years
|
||||
with max `max_gap` years between observations.
|
||||
"""
|
||||
function split_party_years_into_segments(years::Vector{Int}, max_gap::Int)
|
||||
if isempty(years)
|
||||
return Vector{Vector{Int}}()
|
||||
end
|
||||
|
||||
sorted_years = sort(unique(years))
|
||||
segments = [Int[sorted_years[1]]]
|
||||
|
||||
for y in sorted_years[2:end]
|
||||
if y - segments[end][end] <= max_gap
|
||||
push!(segments[end], y)
|
||||
else
|
||||
# Gap too large - start new segment
|
||||
push!(segments, [y])
|
||||
end
|
||||
end
|
||||
|
||||
return segments
|
||||
end
|
||||
|
||||
function prepare_4dim_stan_data(manifesto, expert_dim, expert_lr, year0;
|
||||
union_to_constituents=Dict{Int,Vector{Int}}(),
|
||||
constituent_to_union=Dict{Int,Int}())
|
||||
println("Preparing data for Stan model (Segment-based indexing)...")
|
||||
println(" MAX_GAP = $MAX_GAP years, MIN_OBS = $MIN_OBS observations")
|
||||
|
||||
has_unions = !isempty(union_to_constituents)
|
||||
if has_unions
|
||||
println(" Union mapping: $(length(union_to_constituents)) unions, $(length(constituent_to_union)) constituents")
|
||||
else
|
||||
println(" No union mapping - standard party indexing")
|
||||
end
|
||||
|
||||
# =========================================================================
|
||||
# STEP 1: Collect observation years per party (union-aware)
|
||||
# For union parties: create segments for each CONSTITUENT, not the union.
|
||||
# Union manifesto years are assigned to all constituents.
|
||||
# =========================================================================
|
||||
|
||||
# Identify which party IDs in data are unions vs standalone
|
||||
# NOTE: levels() returns raw types (Int64 for integer party IDs).
|
||||
# We consistently use String keys for all party lookups to avoid type mismatches.
|
||||
manifesto_party_strs = Set(string.(levels(manifesto.party)))
|
||||
union_ids_in_data = Set{String}()
|
||||
if has_unions
|
||||
for uid in keys(union_to_constituents)
|
||||
uid_str = string(uid)
|
||||
if uid_str in manifesto_party_strs
|
||||
push!(union_ids_in_data, uid_str)
|
||||
end
|
||||
end
|
||||
println(" Union party IDs found in manifesto data: $(length(union_ids_in_data))")
|
||||
end
|
||||
|
||||
# Collect all party IDs that need segments
|
||||
# For unions: constituents get segments; union itself does NOT
|
||||
# For standalone: party gets segment as before
|
||||
party_obs_years = Dict{String, Set{Int}}()
|
||||
|
||||
# First pass: collect non-union parties from all data sources (as strings)
|
||||
all_data_parties = Set{String}()
|
||||
for p in levels(manifesto.party)
|
||||
push!(all_data_parties, string(p))
|
||||
end
|
||||
for p in levels(expert_dim.party)
|
||||
push!(all_data_parties, string(p))
|
||||
end
|
||||
for p in levels(expert_lr.party)
|
||||
push!(all_data_parties, string(p))
|
||||
end
|
||||
|
||||
# Initialize observation years for standalone parties and constituents
|
||||
for p in all_data_parties
|
||||
p_int = tryparse(Int, string(p))
|
||||
if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int) && string(p) in union_ids_in_data
|
||||
# This is a union ID in manifesto - skip it, create entries for constituents instead
|
||||
continue
|
||||
end
|
||||
party_obs_years[p] = Set{Int}()
|
||||
end
|
||||
|
||||
# For unions: ensure all constituents have entries
|
||||
if has_unions
|
||||
for (uid, constituents) in union_to_constituents
|
||||
uid_str = string(uid)
|
||||
if uid_str in union_ids_in_data
|
||||
for cid in constituents
|
||||
cid_str = string(cid)
|
||||
if !haskey(party_obs_years, cid_str)
|
||||
party_obs_years[cid_str] = Set{Int}()
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Add years from manifesto
|
||||
for row in eachrow(manifesto)
|
||||
p_str = string(row.party)
|
||||
p_int = tryparse(Int, p_str)
|
||||
if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int) && p_str in union_ids_in_data
|
||||
# Union manifesto: add year to ALL constituents
|
||||
for cid in union_to_constituents[p_int]
|
||||
cid_str = string(cid)
|
||||
if haskey(party_obs_years, cid_str)
|
||||
push!(party_obs_years[cid_str], row.Year)
|
||||
end
|
||||
end
|
||||
else
|
||||
if haskey(party_obs_years, p_str)
|
||||
push!(party_obs_years[p_str], row.Year)
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Add years from expert_dim (individual party data - direct)
|
||||
for row in eachrow(expert_dim)
|
||||
p_str = string(row.party)
|
||||
if haskey(party_obs_years, p_str)
|
||||
push!(party_obs_years[p_str], row.Year)
|
||||
end
|
||||
end
|
||||
|
||||
# Add years from expert_lr (individual party data - direct)
|
||||
for row in eachrow(expert_lr)
|
||||
p_str = string(row.party)
|
||||
if haskey(party_obs_years, p_str)
|
||||
push!(party_obs_years[p_str], row.Year)
|
||||
end
|
||||
end
|
||||
|
||||
J_original = length(party_obs_years)
|
||||
println("Total number of parties to create segments for: $J_original")
|
||||
|
||||
# =========================================================================
|
||||
# STEP 2: Split parties into segments and filter by MIN_OBS
|
||||
# =========================================================================
|
||||
println("\nCreating segments (splitting at gaps > $MAX_GAP years)...")
|
||||
|
||||
segment_data = []
|
||||
segment_id = 0
|
||||
parties_split = 0
|
||||
segments_dropped = 0
|
||||
observations_dropped = 0
|
||||
|
||||
for (p, years_set) in party_obs_years
|
||||
years = collect(years_set)
|
||||
if isempty(years)
|
||||
continue
|
||||
end
|
||||
|
||||
segments = split_party_years_into_segments(years, MAX_GAP)
|
||||
|
||||
if length(segments) > 1
|
||||
parties_split += 1
|
||||
end
|
||||
|
||||
for (seg_num, seg_years) in enumerate(segments)
|
||||
n_obs = length(seg_years)
|
||||
if n_obs >= MIN_OBS
|
||||
segment_id += 1
|
||||
push!(segment_data, (
|
||||
segment_id = segment_id,
|
||||
party_id = p,
|
||||
segment_num = seg_num,
|
||||
year_start = minimum(seg_years),
|
||||
year_end = maximum(seg_years),
|
||||
n_obs = n_obs
|
||||
))
|
||||
else
|
||||
segments_dropped += 1
|
||||
observations_dropped += n_obs
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
segment_info = DataFrame(segment_data)
|
||||
S = nrow(segment_info) # Number of valid segments
|
||||
|
||||
println(" Segments created: $S (from $J_original parties)")
|
||||
println(" Parties split into multiple segments: $parties_split")
|
||||
println(" Segments dropped (< $MIN_OBS obs): $segments_dropped")
|
||||
println(" Observations dropped: $observations_dropped")
|
||||
|
||||
# Get unique parties that have at least one valid segment
|
||||
all_parties = unique(segment_info.party_id)
|
||||
J = length(all_parties)
|
||||
println(" Parties with valid segments: $J")
|
||||
|
||||
# Create party-to-index mapping for the valid parties
|
||||
party_to_index = Dict(all_parties .=> 1:J)
|
||||
|
||||
# =========================================================================
|
||||
# STEP 3: Create segment-year index space (R) - consecutive within segment
|
||||
# =========================================================================
|
||||
println("\nCreating segment-year index space...")
|
||||
|
||||
segment_year_data = []
|
||||
for row in eachrow(segment_info)
|
||||
for y in row.year_start:row.year_end
|
||||
push!(segment_year_data, (
|
||||
segment_id = row.segment_id,
|
||||
party_id = row.party_id,
|
||||
Year = y
|
||||
))
|
||||
end
|
||||
end
|
||||
|
||||
segment_year = DataFrame(segment_year_data)
|
||||
segment_year.rr = 1:nrow(segment_year)
|
||||
R = nrow(segment_year)
|
||||
println("Total segment-year positions (R): $R")
|
||||
|
||||
# Compute len_theta_ts for segments (years per segment)
|
||||
len_theta_ts = [row.year_end - row.year_start + 1 for row in eachrow(segment_info)]
|
||||
@assert sum(len_theta_ts) == R "sum(len_theta_ts)=$(sum(len_theta_ts)) must equal R=$R"
|
||||
|
||||
# =========================================================================
|
||||
# STEP 4: Map observations to segment-year indices (union-aware)
|
||||
# =========================================================================
|
||||
println("\nMapping observations to segment-year indices...")
|
||||
|
||||
# Create lookup: (party_str, year) -> segment_id (for valid segments only)
|
||||
party_year_to_segment = Dict{Tuple{String, Int}, Int}()
|
||||
for row in eachrow(segment_info)
|
||||
for y in row.year_start:row.year_end
|
||||
party_year_to_segment[(string(row.party_id), y)] = row.segment_id
|
||||
end
|
||||
end
|
||||
|
||||
# --- MANIFESTO: union-aware mapping ---
|
||||
# For union manifesto obs: map to first constituent's segment (for ss_man).
|
||||
# The actual theta averaging is handled via constituent arrays.
|
||||
manifesto_segment_ids = Union{Int, Missing}[]
|
||||
for row in eachrow(manifesto)
|
||||
p_str = string(row.party)
|
||||
p_int = tryparse(Int, p_str)
|
||||
key = (p_str, row.Year)
|
||||
|
||||
if haskey(party_year_to_segment, key)
|
||||
# Direct mapping (non-union or constituent with own segment)
|
||||
push!(manifesto_segment_ids, party_year_to_segment[key])
|
||||
elseif p_int !== nothing && has_unions && haskey(union_to_constituents, p_int)
|
||||
# Union party: use first constituent's segment
|
||||
found = false
|
||||
for cid in union_to_constituents[p_int]
|
||||
ckey = (string(cid), row.Year)
|
||||
if haskey(party_year_to_segment, ckey)
|
||||
push!(manifesto_segment_ids, party_year_to_segment[ckey])
|
||||
found = true
|
||||
break
|
||||
end
|
||||
end
|
||||
if !found
|
||||
push!(manifesto_segment_ids, missing)
|
||||
end
|
||||
else
|
||||
push!(manifesto_segment_ids, missing)
|
||||
end
|
||||
end
|
||||
manifesto.segment_id = manifesto_segment_ids
|
||||
n_manifesto_before = nrow(manifesto)
|
||||
manifesto = manifesto[.!ismissing.(manifesto.segment_id), :]
|
||||
manifesto.segment_id = Int.(manifesto.segment_id)
|
||||
println(" Manifesto: $(nrow(manifesto))/$n_manifesto_before observations (dropped $(n_manifesto_before - nrow(manifesto)) in invalid segments)")
|
||||
|
||||
# --- EXPERT DIM: direct mapping (individual party data) ---
|
||||
expert_dim_segment_ids = Union{Int, Missing}[]
|
||||
for row in eachrow(expert_dim)
|
||||
key = (string(row.party), row.Year)
|
||||
if haskey(party_year_to_segment, key)
|
||||
push!(expert_dim_segment_ids, party_year_to_segment[key])
|
||||
else
|
||||
push!(expert_dim_segment_ids, missing)
|
||||
end
|
||||
end
|
||||
expert_dim.segment_id = expert_dim_segment_ids
|
||||
n_expert_dim_before = nrow(expert_dim)
|
||||
expert_dim = expert_dim[.!ismissing.(expert_dim.segment_id), :]
|
||||
expert_dim.segment_id = Int.(expert_dim.segment_id)
|
||||
println(" Expert dim: $(nrow(expert_dim))/$n_expert_dim_before observations (dropped $(n_expert_dim_before - nrow(expert_dim)) in invalid segments)")
|
||||
|
||||
# --- EXPERT LR: direct mapping (individual party data) ---
|
||||
expert_lr_segment_ids = Union{Int, Missing}[]
|
||||
for row in eachrow(expert_lr)
|
||||
key = (string(row.party), row.Year)
|
||||
if haskey(party_year_to_segment, key)
|
||||
push!(expert_lr_segment_ids, party_year_to_segment[key])
|
||||
else
|
||||
push!(expert_lr_segment_ids, missing)
|
||||
end
|
||||
end
|
||||
expert_lr.segment_id = expert_lr_segment_ids
|
||||
n_expert_lr_before = nrow(expert_lr)
|
||||
expert_lr = expert_lr[.!ismissing.(expert_lr.segment_id), :]
|
||||
expert_lr.segment_id = Int.(expert_lr.segment_id)
|
||||
println(" Expert LR: $(nrow(expert_lr))/$n_expert_lr_before observations (dropped $(n_expert_lr_before - nrow(expert_lr)) in invalid segments)")
|
||||
|
||||
# =========================================================================
|
||||
# STEP 5: Create segment indices (ss) for each observation
|
||||
# ss indexes into 1:S (segment space), used for party-level parameters
|
||||
# =========================================================================
|
||||
|
||||
# Create segment_id to ss mapping
|
||||
segment_to_ss = Dict(row.segment_id => i for (i, row) in enumerate(eachrow(segment_info)))
|
||||
|
||||
manifesto.ss_man = [segment_to_ss[sid] for sid in manifesto.segment_id]
|
||||
expert_dim.ss_exp_dim = [segment_to_ss[sid] for sid in expert_dim.segment_id]
|
||||
expert_lr.ss_exp_lr = [segment_to_ss[sid] for sid in expert_lr.segment_id]
|
||||
|
||||
# Validate segment indices
|
||||
@assert all(1 .<= manifesto.ss_man .<= S)
|
||||
@assert all(1 .<= expert_dim.ss_exp_dim .<= S)
|
||||
@assert all(1 .<= expert_lr.ss_exp_lr .<= S)
|
||||
|
||||
# =========================================================================
|
||||
# STEP 6: Map rr indices (segment-year) to datasets via leftjoin
|
||||
# =========================================================================
|
||||
|
||||
# Create (segment_id, Year) -> rr lookup
|
||||
seg_year_to_rr = Dict{Tuple{Int, Int}, Int}()
|
||||
for row in eachrow(segment_year)
|
||||
seg_year_to_rr[(row.segment_id, row.Year)] = row.rr
|
||||
end
|
||||
|
||||
# Join manifesto with segment_year to get rr indices
|
||||
manifesto = leftjoin(manifesto, segment_year, on=[:segment_id, :Year])
|
||||
rename!(manifesto, :rr => :rr_man)
|
||||
|
||||
expert_dim = leftjoin(expert_dim, segment_year, on=[:segment_id, :Year])
|
||||
rename!(expert_dim, :rr => :rr_exp_dim)
|
||||
|
||||
expert_lr = leftjoin(expert_lr, segment_year, on=[:segment_id, :Year])
|
||||
rename!(expert_lr, :rr => :rr_exp_lr)
|
||||
|
||||
# Validate rr mappings
|
||||
@assert all(!ismissing, manifesto.rr_man) "Some manifesto observations have no rr_man mapping"
|
||||
@assert all(!ismissing, expert_dim.rr_exp_dim) "Some expert_dim observations have no rr_exp_dim mapping"
|
||||
@assert all(!ismissing, expert_lr.rr_exp_lr) "Some expert_lr observations have no rr_exp_lr mapping"
|
||||
|
||||
# Convert to Int
|
||||
manifesto.rr_man = Int.(manifesto.rr_man)
|
||||
expert_dim.rr_exp_dim = Int.(expert_dim.rr_exp_dim)
|
||||
expert_lr.rr_exp_lr = Int.(expert_lr.rr_exp_lr)
|
||||
|
||||
# Validate bounds
|
||||
@assert all(1 .<= manifesto.rr_man .<= R) "rr_man out of bounds"
|
||||
@assert all(1 .<= expert_dim.rr_exp_dim .<= R) "rr_exp_dim out of bounds"
|
||||
@assert all(1 .<= expert_lr.rr_exp_lr .<= R) "rr_exp_lr out of bounds"
|
||||
|
||||
# Print diagnostics
|
||||
n_observed = length(unique(vcat(manifesto.rr_man, expert_dim.rr_exp_dim, expert_lr.rr_exp_lr)))
|
||||
println("\nSegment-years with data: $n_observed / $R ($(round(100*n_observed/R, digits=1))%)")
|
||||
println("Segment-years for interpolation: $(R - n_observed)")
|
||||
|
||||
# =========================================================================
|
||||
# STEP 6b: Build constituent arrays for union manifesto/expert observations
|
||||
# For each manifesto obs: store list of constituent rr indices for averaging
|
||||
# Non-union obs: single rr (n_const=1)
|
||||
# Union obs: multiple rr values (n_const=len(constituents))
|
||||
# =========================================================================
|
||||
println("\nBuilding constituent arrays for mean-constituent model...")
|
||||
|
||||
# --- Manifesto constituent arrays ---
|
||||
n_const_man_vec = Int[] # n_const for each manifesto obs
|
||||
const_rr_man_vec = Int[] # flat array of constituent rr values
|
||||
const_offset_man_vec = Int[] # offset into const_rr for each obs
|
||||
|
||||
offset = 1
|
||||
for row in eachrow(manifesto)
|
||||
p_str = string(row.party)
|
||||
p_int = tryparse(Int, p_str)
|
||||
|
||||
if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int) && p_str in union_ids_in_data
|
||||
# Union manifesto obs: find rr for each constituent in this year
|
||||
constituent_rrs = Int[]
|
||||
for cid in union_to_constituents[p_int]
|
||||
ckey = (string(cid), row.Year)
|
||||
if haskey(party_year_to_segment, ckey)
|
||||
sid = party_year_to_segment[ckey]
|
||||
rr_key = (sid, row.Year)
|
||||
if haskey(seg_year_to_rr, rr_key)
|
||||
push!(constituent_rrs, seg_year_to_rr[rr_key])
|
||||
end
|
||||
end
|
||||
end
|
||||
if isempty(constituent_rrs)
|
||||
# Fallback: use the rr_man already assigned
|
||||
push!(constituent_rrs, row.rr_man)
|
||||
end
|
||||
push!(n_const_man_vec, length(constituent_rrs))
|
||||
push!(const_offset_man_vec, offset)
|
||||
append!(const_rr_man_vec, constituent_rrs)
|
||||
offset += length(constituent_rrs)
|
||||
else
|
||||
# Non-union: single constituent (itself)
|
||||
push!(n_const_man_vec, 1)
|
||||
push!(const_offset_man_vec, offset)
|
||||
push!(const_rr_man_vec, row.rr_man)
|
||||
offset += 1
|
||||
end
|
||||
end
|
||||
|
||||
N_const_man_total = length(const_rr_man_vec)
|
||||
n_union_man = count(x -> x > 1, n_const_man_vec)
|
||||
println(" Manifesto: $(nrow(manifesto)) obs, $n_union_man union obs, $N_const_man_total total constituent entries")
|
||||
|
||||
# --- Expert dim constituent arrays ---
|
||||
# Individual expert obs always have n_const=1 (direct mapping)
|
||||
# Union-level expert obs (if any) would average — but typically expert data is at individual party level
|
||||
n_const_exp_dim_vec = Int[]
|
||||
const_rr_exp_dim_vec = Int[]
|
||||
const_offset_exp_dim_vec = Int[]
|
||||
|
||||
offset = 1
|
||||
for row in eachrow(expert_dim)
|
||||
p_str = string(row.party)
|
||||
p_int = tryparse(Int, p_str)
|
||||
|
||||
if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int) && p_str in union_ids_in_data
|
||||
# Union-level expert obs: average over constituents
|
||||
constituent_rrs = Int[]
|
||||
for cid in union_to_constituents[p_int]
|
||||
ckey = (string(cid), row.Year)
|
||||
if haskey(party_year_to_segment, ckey)
|
||||
sid = party_year_to_segment[ckey]
|
||||
rr_key = (sid, row.Year)
|
||||
if haskey(seg_year_to_rr, rr_key)
|
||||
push!(constituent_rrs, seg_year_to_rr[rr_key])
|
||||
end
|
||||
end
|
||||
end
|
||||
if isempty(constituent_rrs)
|
||||
push!(constituent_rrs, row.rr_exp_dim)
|
||||
end
|
||||
push!(n_const_exp_dim_vec, length(constituent_rrs))
|
||||
push!(const_offset_exp_dim_vec, offset)
|
||||
append!(const_rr_exp_dim_vec, constituent_rrs)
|
||||
offset += length(constituent_rrs)
|
||||
else
|
||||
# Individual party obs
|
||||
push!(n_const_exp_dim_vec, 1)
|
||||
push!(const_offset_exp_dim_vec, offset)
|
||||
push!(const_rr_exp_dim_vec, row.rr_exp_dim)
|
||||
offset += 1
|
||||
end
|
||||
end
|
||||
|
||||
N_const_exp_dim_total = length(const_rr_exp_dim_vec)
|
||||
n_union_exp_dim = count(x -> x > 1, n_const_exp_dim_vec)
|
||||
println(" Expert dim: $(nrow(expert_dim)) obs, $n_union_exp_dim union obs, $N_const_exp_dim_total total constituent entries")
|
||||
|
||||
# --- Expert LR constituent arrays ---
|
||||
n_const_exp_lr_vec = Int[]
|
||||
const_rr_exp_lr_vec = Int[]
|
||||
const_offset_exp_lr_vec = Int[]
|
||||
|
||||
offset = 1
|
||||
for row in eachrow(expert_lr)
|
||||
p_str = string(row.party)
|
||||
p_int = tryparse(Int, p_str)
|
||||
|
||||
if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int) && p_str in union_ids_in_data
|
||||
constituent_rrs = Int[]
|
||||
for cid in union_to_constituents[p_int]
|
||||
ckey = (string(cid), row.Year)
|
||||
if haskey(party_year_to_segment, ckey)
|
||||
sid = party_year_to_segment[ckey]
|
||||
rr_key = (sid, row.Year)
|
||||
if haskey(seg_year_to_rr, rr_key)
|
||||
push!(constituent_rrs, seg_year_to_rr[rr_key])
|
||||
end
|
||||
end
|
||||
end
|
||||
if isempty(constituent_rrs)
|
||||
push!(constituent_rrs, row.rr_exp_lr)
|
||||
end
|
||||
push!(n_const_exp_lr_vec, length(constituent_rrs))
|
||||
push!(const_offset_exp_lr_vec, offset)
|
||||
append!(const_rr_exp_lr_vec, constituent_rrs)
|
||||
offset += length(constituent_rrs)
|
||||
else
|
||||
push!(n_const_exp_lr_vec, 1)
|
||||
push!(const_offset_exp_lr_vec, offset)
|
||||
push!(const_rr_exp_lr_vec, row.rr_exp_lr)
|
||||
offset += 1
|
||||
end
|
||||
end
|
||||
|
||||
N_const_exp_lr_total = length(const_rr_exp_lr_vec)
|
||||
n_union_exp_lr = count(x -> x > 1, n_const_exp_lr_vec)
|
||||
println(" Expert LR: $(nrow(expert_lr)) obs, $n_union_exp_lr union obs, $N_const_exp_lr_total total constituent entries")
|
||||
|
||||
# =========================================================================
|
||||
# STEP 7: Filter manifesto items with sufficient observations
|
||||
# =========================================================================
|
||||
var_man_counts_df = combine(groupby(manifesto, :var), :var => length => :n_obs)
|
||||
var_man_counts_df = var_man_counts_df[var_man_counts_df.n_obs .>= 2, :]
|
||||
var_man_counts = var_man_counts_df.var
|
||||
|
||||
var_exp_dim_counts_df = combine(groupby(expert_dim, :var), :var => length => :n_obs)
|
||||
var_exp_dim_counts_df = var_exp_dim_counts_df[var_exp_dim_counts_df.n_obs .>= 2, :]
|
||||
var_exp_dim_counts = var_exp_dim_counts_df.var
|
||||
|
||||
var_exp_lr_counts_df = combine(groupby(expert_lr, :var), :var => length => :n_obs)
|
||||
var_exp_lr_counts_df = var_exp_lr_counts_df[var_exp_lr_counts_df.n_obs .>= 2, :]
|
||||
var_exp_lr_counts = var_exp_lr_counts_df.var
|
||||
|
||||
manifesto = manifesto[in.(manifesto.var, Ref(var_man_counts)), :]
|
||||
manifesto.var_man = levelcode.(categorical(manifesto.var, levels=unique(var_man_counts)))
|
||||
|
||||
expert_dim = expert_dim[in.(expert_dim.var, Ref(var_exp_dim_counts)), :]
|
||||
expert_dim.var_exp_dim = levelcode.(categorical(expert_dim.var, levels=unique(var_exp_dim_counts)))
|
||||
|
||||
expert_lr = expert_lr[in.(expert_lr.var, Ref(var_exp_lr_counts)), :]
|
||||
expert_lr.var_exp_lr = levelcode.(categorical(expert_lr.var, levels=unique(var_exp_lr_counts)))
|
||||
|
||||
# =========================================================================
|
||||
# STEP 8: Country (group) indexing
|
||||
# =========================================================================
|
||||
all_groups = unique(vcat(levels(manifesto.country), levels(expert_dim.country), levels(expert_lr.country)))
|
||||
P = length(all_groups)
|
||||
println("Total number of unique countries (P): $P")
|
||||
|
||||
group_to_index = Dict(all_groups .=> 1:P)
|
||||
manifesto.pp_man = [group_to_index[c] for c in manifesto.country]
|
||||
expert_dim.pp_exp_dim = [group_to_index[c] for c in expert_dim.country]
|
||||
expert_lr.pp_exp_lr = [group_to_index[c] for c in expert_lr.country]
|
||||
|
||||
@assert all(1 .<= manifesto.pp_man .<= P)
|
||||
@assert all(1 .<= expert_dim.pp_exp_dim .<= P)
|
||||
@assert all(1 .<= expert_lr.pp_exp_lr .<= P)
|
||||
|
||||
# =========================================================================
|
||||
# STEP 9: Country-item-year combinations for zero-inflation model
|
||||
# =========================================================================
|
||||
manifesto.ciy_key = string.(manifesto.country, "_", manifesto.var_man, "_", manifesto.Year)
|
||||
ciy_keys = unique(manifesto.ciy_key)
|
||||
N_ciy = length(ciy_keys)
|
||||
println("Total unique country-item-year combinations (N_ciy): $N_ciy")
|
||||
|
||||
ciy_key_to_index = Dict(ciy_keys .=> 1:N_ciy)
|
||||
manifesto.ciy_idx = [ciy_key_to_index[k] for k in manifesto.ciy_key]
|
||||
@assert all(1 .<= manifesto.ciy_idx .<= N_ciy)
|
||||
|
||||
# =========================================================================
|
||||
# STEP 10: Segment-country mapping (each segment inherits from party)
|
||||
# For constituents: inherit country from their union's manifesto data
|
||||
# =========================================================================
|
||||
party_country_dict = Dict{String, Int}()
|
||||
|
||||
# From data directly
|
||||
for row in eachrow(manifesto)
|
||||
p_str = string(row.party)
|
||||
p_int = tryparse(Int, p_str)
|
||||
c_idx = group_to_index[string(row.country)]
|
||||
party_country_dict[p_str] = c_idx
|
||||
|
||||
# If union, also assign country to all constituents
|
||||
if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int)
|
||||
for cid in union_to_constituents[p_int]
|
||||
party_country_dict[string(cid)] = c_idx
|
||||
end
|
||||
end
|
||||
end
|
||||
for row in eachrow(expert_dim)
|
||||
party_country_dict[string(row.party)] = group_to_index[string(row.country)]
|
||||
end
|
||||
for row in eachrow(expert_lr)
|
||||
party_country_dict[string(row.party)] = group_to_index[string(row.country)]
|
||||
end
|
||||
|
||||
# Map each segment to its party's country
|
||||
segment_country_idx = Int[]
|
||||
for row in eachrow(segment_info)
|
||||
pid = string(row.party_id)
|
||||
if haskey(party_country_dict, pid)
|
||||
push!(segment_country_idx, party_country_dict[pid])
|
||||
else
|
||||
# Fallback: try to find via union mapping
|
||||
pf_int = tryparse(Int, pid)
|
||||
if pf_int !== nothing && haskey(constituent_to_union, pf_int)
|
||||
uid = constituent_to_union[pf_int]
|
||||
if haskey(party_country_dict, string(uid))
|
||||
push!(segment_country_idx, party_country_dict[string(uid)])
|
||||
else
|
||||
error("Cannot find country for constituent $pid (union $uid)")
|
||||
end
|
||||
else
|
||||
error("Cannot find country for party $pid")
|
||||
end
|
||||
end
|
||||
end
|
||||
@assert all(1 .<= segment_country_idx .<= P)
|
||||
@assert length(segment_country_idx) == S
|
||||
|
||||
# Persist canonical country on segment mapping tables
|
||||
segment_country = [all_groups[idx] for idx in segment_country_idx]
|
||||
segment_info.country = segment_country
|
||||
|
||||
segment_country_by_id = Dict(row.segment_id => segment_country[i] for (i, row) in enumerate(eachrow(segment_info)))
|
||||
segment_year.country = [segment_country_by_id[sid] for sid in segment_year.segment_id]
|
||||
|
||||
# =========================================================================
|
||||
# STEP 11: Load party family data and map to segments
|
||||
# =========================================================================
|
||||
println("\nLoading party family data...")
|
||||
party_families_file = joinpath("data", "party_families.csv")
|
||||
if !isfile(party_families_file)
|
||||
error("party_families.csv not found at $party_families_file")
|
||||
end
|
||||
party_families_df = CSV.read(party_families_file, DataFrame)
|
||||
|
||||
pf_to_family = Dict(row.partyfacts_id => row.family for row in eachrow(party_families_df))
|
||||
|
||||
all_family_names = unique(party_families_df.family)
|
||||
family_to_idx = Dict(f => i for (i, f) in enumerate(all_family_names))
|
||||
F = length(all_family_names)
|
||||
println("Total number of party families (F): $F")
|
||||
println("Family categories: ", join(all_family_names, ", "))
|
||||
|
||||
# Map each segment to its party's family
|
||||
# For constituents: try own family first, fall back to union's family
|
||||
segment_family_idx = Int[]
|
||||
unmatched_segments = String[]
|
||||
default_family_idx = haskey(family_to_idx, "other") ? family_to_idx["other"] : 1
|
||||
|
||||
for row in eachrow(segment_info)
|
||||
pf_id = tryparse(Int, string(row.party_id))
|
||||
if !isnothing(pf_id) && haskey(pf_to_family, pf_id)
|
||||
family_name = pf_to_family[pf_id]
|
||||
push!(segment_family_idx, family_to_idx[family_name])
|
||||
elseif !isnothing(pf_id) && haskey(constituent_to_union, pf_id) && haskey(pf_to_family, constituent_to_union[pf_id])
|
||||
# Fall back to union's family
|
||||
family_name = pf_to_family[constituent_to_union[pf_id]]
|
||||
push!(segment_family_idx, family_to_idx[family_name])
|
||||
else
|
||||
push!(segment_family_idx, default_family_idx)
|
||||
push!(unmatched_segments, "$(row.party_id)_seg$(row.segment_num)")
|
||||
end
|
||||
end
|
||||
|
||||
if !isempty(unmatched_segments)
|
||||
println(" Warning: $(length(unmatched_segments)) segments not matched to families (assigned to 'other')")
|
||||
if length(unmatched_segments) <= 10
|
||||
println(" Unmatched: ", join(unmatched_segments, ", "))
|
||||
end
|
||||
end
|
||||
|
||||
@assert all(1 .<= segment_family_idx .<= F)
|
||||
@assert length(segment_family_idx) == S
|
||||
|
||||
# =========================================================================
|
||||
# STEP 12: Find anchor segment
|
||||
# With unions: use CDU (1375) as anchor (individual constituent)
|
||||
# Without unions: use CDU/CSU (211) as anchor (union ID)
|
||||
# =========================================================================
|
||||
anchor_party_id = has_unions ? 1375 : 211
|
||||
anchor_label = has_unions ? "CDU" : "CDU/CSU"
|
||||
anchor_segments = filter(row -> tryparse(Int, string(row.party_id)) == anchor_party_id, segment_info)
|
||||
|
||||
if nrow(anchor_segments) > 0
|
||||
# Pick segment with most observations
|
||||
anchor_segment_idx = anchor_segments[argmax(anchor_segments.n_obs), :segment_id]
|
||||
# Convert to ss index (1:S)
|
||||
anchor_segment_ss = segment_to_ss[anchor_segment_idx]
|
||||
println(" Anchor segment ($anchor_label, ID $anchor_party_id): segment $anchor_segment_ss ($(anchor_segments[argmax(anchor_segments.n_obs), :n_obs]) obs)")
|
||||
else
|
||||
println(" Warning: Anchor party $anchor_label (ID $anchor_party_id) not found, using segment 1")
|
||||
anchor_segment_ss = 1
|
||||
end
|
||||
|
||||
# =========================================================================
|
||||
# STEP 13: Validation - check no long gaps remain within segments
|
||||
# =========================================================================
|
||||
println("\nValidating segment structure...")
|
||||
rr_to_segment_year = Dict(row.rr => (segment_id=row.segment_id, year=row.Year) for row in eachrow(segment_year))
|
||||
seg_obs_years = Dict(row.segment_id => Set{Int}() for row in eachrow(segment_info))
|
||||
|
||||
for row in eachrow(manifesto)
|
||||
push!(seg_obs_years[row.segment_id], row.Year)
|
||||
end
|
||||
for row in eachrow(expert_dim)
|
||||
push!(seg_obs_years[row.segment_id], row.Year)
|
||||
end
|
||||
for row in eachrow(expert_lr)
|
||||
push!(seg_obs_years[row.segment_id], row.Year)
|
||||
end
|
||||
|
||||
# Union manifesto rows contribute to every constituent through const_rr_man_vec,
|
||||
# even though row.segment_id stores only a representative segment for indexing.
|
||||
# Include these constituent rr values so validation matches the actual Stan data.
|
||||
for rr in const_rr_man_vec
|
||||
if haskey(rr_to_segment_year, rr)
|
||||
sy = rr_to_segment_year[rr]
|
||||
push!(seg_obs_years[sy.segment_id], sy.year)
|
||||
end
|
||||
end
|
||||
|
||||
max_internal_gap = 0
|
||||
for (i, row) in enumerate(eachrow(segment_info))
|
||||
seg_obs = collect(seg_obs_years[row.segment_id])
|
||||
if length(seg_obs) > 1
|
||||
gaps = diff(sort(unique(seg_obs)))
|
||||
if !isempty(gaps)
|
||||
max_gap_in_seg = maximum(gaps)
|
||||
max_internal_gap = max(max_internal_gap, max_gap_in_seg)
|
||||
if max_gap_in_seg > MAX_GAP
|
||||
@warn "Segment $i (party $(row.party_id)) has internal gap of $max_gap_in_seg years!"
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
println(" Maximum internal gap within segments: $max_internal_gap years (limit: $MAX_GAP)")
|
||||
|
||||
# Validate all segments have >= MIN_OBS
|
||||
for row in eachrow(segment_info)
|
||||
@assert row.n_obs >= MIN_OBS "Segment $(row.segment_id) has only $(row.n_obs) obs (min: $MIN_OBS)"
|
||||
end
|
||||
println(" All segments have >= $MIN_OBS observations: PASS")
|
||||
|
||||
println("\nSegment-based indexing created successfully")
|
||||
println(" S (segments): $S")
|
||||
println(" R (segment-years): $R")
|
||||
println(" J (parties with valid segments): $J")
|
||||
|
||||
return (manifesto=manifesto, expert_dim=expert_dim, expert_lr=expert_lr,
|
||||
segment_year=segment_year, segment_info=segment_info,
|
||||
all_parties=all_parties, all_groups=all_groups,
|
||||
S=S, J=J, P=P, R=R, N_ciy=N_ciy, len_theta_ts=len_theta_ts,
|
||||
segment_country_idx=segment_country_idx, group_to_index=group_to_index,
|
||||
F=F, segment_family_idx=segment_family_idx, anchor_segment_idx=anchor_segment_ss,
|
||||
# Constituent arrays for mean-constituent model
|
||||
N_const_man_total=N_const_man_total,
|
||||
n_const_man=n_const_man_vec,
|
||||
const_offset_man=const_offset_man_vec,
|
||||
const_rr_man=const_rr_man_vec,
|
||||
N_const_exp_dim_total=N_const_exp_dim_total,
|
||||
n_const_exp_dim=n_const_exp_dim_vec,
|
||||
const_offset_exp_dim=const_offset_exp_dim_vec,
|
||||
const_rr_exp_dim=const_rr_exp_dim_vec,
|
||||
N_const_exp_lr_total=N_const_exp_lr_total,
|
||||
n_const_exp_lr=n_const_exp_lr_vec,
|
||||
const_offset_exp_lr=const_offset_exp_lr_vec,
|
||||
const_rr_exp_lr=const_rr_exp_lr_vec,
|
||||
union_to_constituents=union_to_constituents,
|
||||
constituent_to_union=constituent_to_union)
|
||||
end
|
||||
|
||||
function finalize_4dim_stan_data(manifesto, expert_dim, expert_lr, segment_year, segment_info,
|
||||
all_parties, all_groups, group_to_index, year0, S, J, P, R, N_ciy,
|
||||
len_theta_ts, segment_country_idx, F, segment_family_idx, anchor_segment_idx;
|
||||
N_const_man_total=0, n_const_man=Int[], const_offset_man=Int[], const_rr_man=Int[],
|
||||
N_const_exp_dim_total=0, n_const_exp_dim=Int[], const_offset_exp_dim=Int[], const_rr_exp_dim=Int[],
|
||||
N_const_exp_lr_total=0, n_const_exp_lr=Int[], const_offset_exp_lr=Int[], const_rr_exp_lr=Int[])
|
||||
println("Finalizing 4D Stan data structure (V10: segment-based)...")
|
||||
|
||||
# Map years for temporal indexing
|
||||
years_all = sort(unique(vcat(manifesto.Year, expert_dim.Year, expert_lr.Year)))
|
||||
T_year = length(years_all)
|
||||
year_map = DataFrame(year_rel=years_all, year_ix=1:T_year)
|
||||
|
||||
# Apply year mapping to all datasets
|
||||
manifesto = leftjoin(manifesto, year_map, on=[:Year => :year_rel])
|
||||
rename!(manifesto, :year_ix => :year_for_man)
|
||||
|
||||
expert_dim = leftjoin(expert_dim, year_map, on=[:Year => :year_rel])
|
||||
rename!(expert_dim, :year_ix => :year_for_exp_dim)
|
||||
|
||||
expert_lr = leftjoin(expert_lr, year_map, on=[:Year => :year_rel])
|
||||
rename!(expert_lr, :year_ix => :year_for_exp_lr)
|
||||
|
||||
# Validate year assignments
|
||||
@assert all(.!ismissing.(manifesto.year_for_man))
|
||||
@assert all(.!ismissing.(expert_dim.year_for_exp_dim))
|
||||
@assert all(.!ismissing.(expert_lr.year_for_exp_lr))
|
||||
@assert all(1 .<= manifesto.year_for_man .<= T_year)
|
||||
@assert all(1 .<= expert_dim.year_for_exp_dim .<= T_year)
|
||||
@assert all(1 .<= expert_lr.year_for_exp_lr .<= T_year)
|
||||
|
||||
# Add small epsilon to prevent exact zeros and ones
|
||||
epsilon = 1e-6
|
||||
expert_dim.val = clamp.(expert_dim.val, epsilon, 1.0 - epsilon)
|
||||
expert_lr.val = clamp.(expert_lr.val, epsilon, 1.0 - epsilon)
|
||||
|
||||
# Calculate prior means
|
||||
man_positive_sample = manifesto.positive[manifesto.sample .> 0] ./ manifesto.sample[manifesto.sample .> 0]
|
||||
mn_resp_log_man = StatsFuns.logit(mean(man_positive_sample))
|
||||
mn_resp_log_exp_dim = StatsFuns.logit(mean(expert_dim.val))
|
||||
mn_resp_log_exp_lr = StatsFuns.logit(mean(expert_lr.val))
|
||||
|
||||
println("Prior means calculated:")
|
||||
println(" Manifesto: $(round(mn_resp_log_man, digits=3))")
|
||||
println(" Expert dimension-specific: $(round(mn_resp_log_exp_dim, digits=3))")
|
||||
println(" Expert general L-R: $(round(mn_resp_log_exp_lr, digits=3))")
|
||||
|
||||
# V6: Decade indexing for hierarchical L-R weights
|
||||
expert_lr_actual_years = expert_lr.Year .+ year0
|
||||
expert_lr_decade_raw = div.(expert_lr_actual_years, 10)
|
||||
all_lr_decades = sort(unique(expert_lr_decade_raw))
|
||||
lr_decade_to_index = Dict(all_lr_decades .=> 1:length(all_lr_decades))
|
||||
dd_exp_lr = [lr_decade_to_index[d] for d in expert_lr_decade_raw]
|
||||
D_lr = length(all_lr_decades)
|
||||
println(" Decade indexing (V6): $D_lr decades, range $(minimum(all_lr_decades)*10)s-$(maximum(all_lr_decades)*10)s")
|
||||
|
||||
# Create Stan data dictionary - V10 uses S (segments) instead of J (parties)
|
||||
dat_4dim = Dict(
|
||||
# Common data - V10: S = number of segments
|
||||
"S" => S, # NEW: Number of segments (was J)
|
||||
"J" => J, # Keep J for reference (parties with valid segments)
|
||||
"P" => P,
|
||||
"R" => R,
|
||||
"T_year" => T_year,
|
||||
"len_theta_ts" => Int.(len_theta_ts),
|
||||
|
||||
# Segment-country mapping (V10: segments inherit country from party)
|
||||
"segment_country" => segment_country_idx,
|
||||
|
||||
# Segment family data (V10: segments inherit family from party)
|
||||
"F" => F,
|
||||
"segment_family" => segment_family_idx,
|
||||
|
||||
# Anchor segment for identification (CDU/CSU segment)
|
||||
"anchor_segment" => anchor_segment_idx,
|
||||
|
||||
# Manifesto data - use ss_man (segment index) instead of jj_man (party index)
|
||||
"N_man" => nrow(manifesto),
|
||||
"K_man" => length(unique(manifesto.var_man)),
|
||||
"kk_man" => manifesto.var_man,
|
||||
"ss_man" => manifesto.ss_man, # V10: segment index (was jj_man)
|
||||
"rr_man" => manifesto.rr_man,
|
||||
"pp_man" => manifesto.pp_man,
|
||||
"positive" => manifesto.positive,
|
||||
"sample" => manifesto.sample,
|
||||
"year_for_man" => manifesto.year_for_man,
|
||||
"type_high_idx_man" => manifesto.type_high_idx,
|
||||
"type_low_idx_man" => manifesto.type_low_idx,
|
||||
|
||||
# V1 (2D model): dimension index and direction for text data
|
||||
"dim_idx_man" => manifesto.dim_idx_man,
|
||||
"direction_man" => manifesto.direction_man,
|
||||
|
||||
# Country-item-year data for zero-inflation
|
||||
"N_ciy" => N_ciy,
|
||||
"ciy_idx" => manifesto.ciy_idx,
|
||||
|
||||
# Expert dimension-specific data
|
||||
"N_exp_dim" => nrow(expert_dim),
|
||||
"K_exp_dim" => length(unique(expert_dim.var_exp_dim)),
|
||||
"kk_exp_dim" => expert_dim.var_exp_dim,
|
||||
"ss_exp_dim" => expert_dim.ss_exp_dim, # V10: segment index
|
||||
"rr_exp_dim" => expert_dim.rr_exp_dim,
|
||||
"pp_exp_dim" => expert_dim.pp_exp_dim,
|
||||
# V5 K-scaling: use rounded sum (mean × K × n_scale) and total trials (K × n_scale)
|
||||
"val_dim_int" => Int.(clamp.(round.(expert_dim.val .* expert_dim.n_scale .* expert_dim.n_experts), 0, expert_dim.n_scale .* expert_dim.n_experts)),
|
||||
"n_total_exp_dim" => expert_dim.n_scale .* expert_dim.n_experts,
|
||||
"n_experts_exp_dim" => expert_dim.n_experts,
|
||||
"type_high_idx" => expert_dim.type_high_idx,
|
||||
"type_low_idx" => expert_dim.type_low_idx,
|
||||
|
||||
# V1 (2D model): dimension index for expert dimension data
|
||||
"dim_idx_exp" => expert_dim.dim_idx_exp,
|
||||
|
||||
# Expert general L-R data
|
||||
"N_exp_lr" => nrow(expert_lr),
|
||||
"K_exp_lr" => length(unique(expert_lr.var_exp_lr)),
|
||||
"kk_exp_lr" => expert_lr.var_exp_lr,
|
||||
"ss_exp_lr" => expert_lr.ss_exp_lr, # V10: segment index
|
||||
"rr_exp_lr" => expert_lr.rr_exp_lr,
|
||||
"pp_exp_lr" => expert_lr.pp_exp_lr,
|
||||
# V5 K-scaling: use rounded sum (mean × K × n_scale) and total trials (K × n_scale)
|
||||
"val_lr_int" => Int.(clamp.(round.(expert_lr.val .* expert_lr.n_scale .* expert_lr.n_experts), 0, expert_lr.n_scale .* expert_lr.n_experts)),
|
||||
"n_total_exp_lr" => expert_lr.n_scale .* expert_lr.n_experts,
|
||||
"n_experts_exp_lr" => expert_lr.n_experts,
|
||||
|
||||
# V6: Decade indexing for hierarchical L-R weights
|
||||
"D_lr" => D_lr,
|
||||
"dd_exp_lr" => dd_exp_lr,
|
||||
|
||||
# Prior information
|
||||
"mn_resp_log_man" => mn_resp_log_man,
|
||||
"mn_resp_log_exp_dim" => mn_resp_log_exp_dim,
|
||||
"mn_resp_log_exp_lr" => mn_resp_log_exp_lr,
|
||||
|
||||
# Constituent arrays for mean-constituent model (V4)
|
||||
"N_const_man_total" => max(1, N_const_man_total),
|
||||
"n_const_man" => isempty(n_const_man) ? ones(Int, nrow(manifesto)) : n_const_man,
|
||||
"const_offset_man" => isempty(const_offset_man) ? collect(1:nrow(manifesto)) : const_offset_man,
|
||||
"const_rr_man" => isempty(const_rr_man) ? manifesto.rr_man : const_rr_man,
|
||||
|
||||
"N_const_exp_dim_total" => max(1, N_const_exp_dim_total),
|
||||
"n_const_exp_dim" => isempty(n_const_exp_dim) ? ones(Int, nrow(expert_dim)) : n_const_exp_dim,
|
||||
"const_offset_exp_dim" => isempty(const_offset_exp_dim) ? collect(1:nrow(expert_dim)) : const_offset_exp_dim,
|
||||
"const_rr_exp_dim" => isempty(const_rr_exp_dim) ? expert_dim.rr_exp_dim : const_rr_exp_dim,
|
||||
|
||||
"N_const_exp_lr_total" => max(1, N_const_exp_lr_total),
|
||||
"n_const_exp_lr" => isempty(n_const_exp_lr) ? ones(Int, nrow(expert_lr)) : n_const_exp_lr,
|
||||
"const_offset_exp_lr" => isempty(const_offset_exp_lr) ? collect(1:nrow(expert_lr)) : const_offset_exp_lr,
|
||||
"const_rr_exp_lr" => isempty(const_rr_exp_lr) ? expert_lr.rr_exp_lr : const_rr_exp_lr
|
||||
)
|
||||
|
||||
println("4D Stan data dictionary created with $(length(dat_4dim)) elements")
|
||||
|
||||
# Print summary statistics
|
||||
println("\nData summary (V10: Segment-based):")
|
||||
println(" Segments: $(dat_4dim["S"])")
|
||||
println(" Parties with valid segments: $(dat_4dim["J"])")
|
||||
println(" Segment-year combinations: $(dat_4dim["R"])")
|
||||
println(" Manifesto observations: $(dat_4dim["N_man"])")
|
||||
println(" Expert dimension-specific observations: $(dat_4dim["N_exp_dim"])")
|
||||
println(" Expert general L-R observations: $(dat_4dim["N_exp_lr"])")
|
||||
println(" Unique manifesto items: $(dat_4dim["K_man"])")
|
||||
println(" Unique expert dimension-specific items: $(dat_4dim["K_exp_dim"])")
|
||||
println(" Unique expert general L-R items: $(dat_4dim["K_exp_lr"])")
|
||||
println(" Years: $(dat_4dim["T_year"])")
|
||||
|
||||
return (dat_4dim=dat_4dim, manifesto=manifesto, expert_dim=expert_dim,
|
||||
expert_lr=expert_lr, T_year=T_year, segment_year=segment_year,
|
||||
segment_info=segment_info)
|
||||
end
|
||||
|
||||
# Execute if run directly
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
println("Run from main script to execute the full 4D pipeline")
|
||||
end
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,174 @@
|
||||
#!/usr/bin/env julia
|
||||
#############################################################################
|
||||
## 05_results_processing.jl
|
||||
## Extract and process 4D model results with diagnostics
|
||||
## Adapted from old_project for latent traits only (no election effects)
|
||||
#############################################################################
|
||||
|
||||
using StanSample, DataFrames, Statistics
|
||||
|
||||
function extract_model_results_4dim(stanmodel)
|
||||
"""
|
||||
Extract model results for 4D latent trait model
|
||||
Simplified version - no election effects (pure latent traits)
|
||||
"""
|
||||
println("Extracting 4D model results...")
|
||||
|
||||
try
|
||||
println("Model completed successfully - extracting results")
|
||||
|
||||
# For 4D latent trait model, we save the full stanmodel object
|
||||
# Post-estimation will extract specific parameters later
|
||||
|
||||
return (
|
||||
samples = stanmodel, # Save the full stanmodel with all MCMC samples
|
||||
extraction_status = "success"
|
||||
)
|
||||
|
||||
catch e
|
||||
println("Error in result extraction: $e")
|
||||
return (
|
||||
samples = "error",
|
||||
extraction_status = "error",
|
||||
error_message = string(e)
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
function compute_model_diagnostics(stanmodel_result)
|
||||
"""
|
||||
Compute convergence diagnostics from Stan model
|
||||
Returns R-hat, ESS statistics, and overall convergence assessment
|
||||
|
||||
stanmodel_result can be either:
|
||||
- A SampleModel object directly
|
||||
- A named tuple from run_4dim_stan_model containing .stanmodel
|
||||
"""
|
||||
println("Computing model diagnostics...")
|
||||
|
||||
try
|
||||
# Handle both direct SampleModel and named tuple from run_4dim_stan_model
|
||||
stanmodel = if hasproperty(stanmodel_result, :stanmodel)
|
||||
stanmodel_result.stanmodel
|
||||
else
|
||||
stanmodel_result
|
||||
end
|
||||
|
||||
# Get REAL diagnostics using StanSample.jl
|
||||
diagnostics_summary = read_summary(stanmodel)
|
||||
|
||||
# Extract real Rhat and ESS values. Stan summary column names differ
|
||||
# across CmdStan/StanSample versions, so resolve aliases explicitly.
|
||||
summary_names = names(diagnostics_summary)
|
||||
rhat_col = if "r_hat" in summary_names
|
||||
"r_hat"
|
||||
elseif "R_hat" in summary_names
|
||||
"R_hat"
|
||||
elseif "RHat" in summary_names
|
||||
"RHat"
|
||||
else
|
||||
error("No R-hat column found in summary. Columns: $(join(summary_names, ", "))")
|
||||
end
|
||||
ess_col = if "ess_bulk" in summary_names
|
||||
"ess_bulk"
|
||||
elseif "ess" in summary_names
|
||||
"ess"
|
||||
elseif "ESS_bulk" in summary_names
|
||||
"ESS_bulk"
|
||||
elseif "n_eff" in summary_names
|
||||
"n_eff"
|
||||
else
|
||||
error("No ESS column found in summary. Columns: $(join(summary_names, ", "))")
|
||||
end
|
||||
|
||||
rhat_vals = diagnostics_summary[!, rhat_col]
|
||||
ess_bulk_vals = diagnostics_summary[!, ess_col]
|
||||
|
||||
# Compute real statistics (handle NaN values properly)
|
||||
# Use isfinite to exclude both missing and NaN values
|
||||
valid_rhat = filter(isfinite, rhat_vals)
|
||||
valid_ess = filter(isfinite, ess_bulk_vals)
|
||||
|
||||
mean_rhat = length(valid_rhat) > 0 ? mean(valid_rhat) : NaN
|
||||
max_rhat = length(valid_rhat) > 0 ? maximum(valid_rhat) : NaN
|
||||
mean_ess = length(valid_ess) > 0 ? mean(valid_ess) : NaN
|
||||
min_ess = length(valid_ess) > 0 ? minimum(valid_ess) : NaN
|
||||
|
||||
# Count problematic parameters (use isfinite for consistent counting)
|
||||
high_rhat_count = count(x -> isfinite(x) && x > 1.1, rhat_vals)
|
||||
moderate_rhat_count = count(x -> isfinite(x) && x > 1.05, rhat_vals)
|
||||
low_ess_count = count(x -> isfinite(x) && x < 400, ess_bulk_vals)
|
||||
very_low_ess_count = count(x -> isfinite(x) && x < 100, ess_bulk_vals)
|
||||
|
||||
# Total parameter count
|
||||
total_params = length(valid_rhat)
|
||||
|
||||
# Overall assessment (handle NaN values)
|
||||
if isnan(max_rhat) || total_params == 0
|
||||
convergence_status = "insufficient_data"
|
||||
else
|
||||
excellent_convergence = max_rhat < 1.05 && high_rhat_count == 0 && very_low_ess_count == 0
|
||||
good_convergence = max_rhat < 1.1 && high_rhat_count < 5 && very_low_ess_count < total_params * 0.1
|
||||
acceptable_convergence = max_rhat < 1.2 && high_rhat_count < total_params * 0.1
|
||||
|
||||
if excellent_convergence
|
||||
convergence_status = "excellent"
|
||||
elseif good_convergence
|
||||
convergence_status = "good"
|
||||
elseif acceptable_convergence
|
||||
convergence_status = "acceptable"
|
||||
else
|
||||
convergence_status = "poor"
|
||||
end
|
||||
end
|
||||
|
||||
println("\nDiagnostics computed:")
|
||||
println(" Total parameters: $total_params")
|
||||
println(" Mean R-hat: $(round(mean_rhat, digits=4))")
|
||||
println(" Max R-hat: $(round(max_rhat, digits=4))")
|
||||
println(" High R-hat count (>1.1): $high_rhat_count")
|
||||
println(" Mean ESS: $(round(mean_ess, digits=0))")
|
||||
println(" Min ESS: $(round(min_ess, digits=0))")
|
||||
println(" Very low ESS count (<100): $very_low_ess_count")
|
||||
println(" Convergence status: $convergence_status")
|
||||
|
||||
return (
|
||||
diagnostics_summary = diagnostics_summary,
|
||||
convergence_status = convergence_status,
|
||||
mean_rhat = mean_rhat,
|
||||
max_rhat = max_rhat,
|
||||
mean_ess = mean_ess,
|
||||
min_ess = min_ess,
|
||||
high_rhat_count = high_rhat_count,
|
||||
moderate_rhat_count = moderate_rhat_count,
|
||||
low_ess_count = low_ess_count,
|
||||
very_low_ess_count = very_low_ess_count,
|
||||
total_params = total_params
|
||||
)
|
||||
|
||||
catch e
|
||||
println("Error in diagnostics computation: $e")
|
||||
println("Stack trace:")
|
||||
showerror(stdout, e, catch_backtrace())
|
||||
|
||||
return (
|
||||
diagnostics_summary = "error",
|
||||
convergence_status = "error",
|
||||
mean_rhat = 999.0,
|
||||
max_rhat = 999.0,
|
||||
mean_ess = 0.0,
|
||||
min_ess = 0.0,
|
||||
high_rhat_count = 999,
|
||||
moderate_rhat_count = 999,
|
||||
low_ess_count = 999,
|
||||
very_low_ess_count = 999,
|
||||
total_params = 0,
|
||||
error_message = string(e)
|
||||
)
|
||||
end
|
||||
end
|
||||
|
||||
# Execute if run directly
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
println("Run from main run_model.jl to execute the full pipeline")
|
||||
end
|
||||
@@ -0,0 +1,428 @@
|
||||
#!/usr/bin/env julia
|
||||
#############################################################################
|
||||
## 06_save_model.jl
|
||||
## CSV-First Save Architecture
|
||||
## Robust, portable, no serialization issues
|
||||
#############################################################################
|
||||
|
||||
module RobustSave
|
||||
|
||||
using Dates, Printf, CSV, DataFrames, JSON
|
||||
|
||||
export robust_save_model_csv, robust_save_model
|
||||
|
||||
"""
|
||||
CSV-First Save Architecture
|
||||
|
||||
When chains_already_saved=true (BULLETPROOF MODE):
|
||||
- Chains already saved to run_dir/chains/ by model execution
|
||||
- Just verify chains and add metadata/data files
|
||||
- Even if this function crashes, chains are SAFE
|
||||
|
||||
When chains_already_saved=false (legacy mode):
|
||||
- Copy chains from temp directory to new run directory
|
||||
- Add metadata/data files
|
||||
|
||||
Saves model results as:
|
||||
1. CSV chain files (source of truth - never fail)
|
||||
2. Data CSVs (original inputs for reproducibility)
|
||||
3. Simple metadata.json (no complex types)
|
||||
4. Human-readable README.txt
|
||||
|
||||
No JLD2, no serialization issues, fully portable and reproducible.
|
||||
"""
|
||||
function robust_save_model_csv(
|
||||
run_dir_or_temp::String,
|
||||
data_dict::Dict,
|
||||
original_data::Dict,
|
||||
metadata::Dict;
|
||||
chains_already_saved::Bool=false
|
||||
)
|
||||
println("\n" * "=" ^ 70)
|
||||
if chains_already_saved
|
||||
println("ADDING METADATA TO EXISTING RUN (chains already secured)")
|
||||
else
|
||||
println("CSV-FIRST MODEL SAVE")
|
||||
end
|
||||
println("=" ^ 70)
|
||||
|
||||
# Determine directories based on mode
|
||||
if chains_already_saved
|
||||
# Chains already saved - run_dir_or_temp IS the run directory
|
||||
run_dir = run_dir_or_temp
|
||||
chains_dir = joinpath(run_dir, "chains")
|
||||
data_dir = joinpath(run_dir, "data")
|
||||
run_id = basename(run_dir)
|
||||
timestamp = replace(run_id, "run_" => "")
|
||||
|
||||
println("Run directory: $run_dir")
|
||||
println("Mode: Chains already secured, adding metadata")
|
||||
|
||||
# Verify chains directory exists
|
||||
if !isdir(chains_dir)
|
||||
error("CRITICAL: Chains directory not found: $chains_dir")
|
||||
end
|
||||
|
||||
# Count existing chain files
|
||||
chain_files = filter(f -> endswith(f, ".csv") && contains(f, "chain"), readdir(chains_dir))
|
||||
if isempty(chain_files)
|
||||
error("CRITICAL: No chain CSV files found in $chains_dir")
|
||||
end
|
||||
|
||||
println("Found $(length(chain_files)) chain files already saved")
|
||||
|
||||
# Calculate total size
|
||||
total_size_gb = 0.0
|
||||
for chain_file in chain_files
|
||||
chain_path = joinpath(chains_dir, chain_file)
|
||||
total_size_gb += filesize(chain_path) / (1024^3)
|
||||
end
|
||||
|
||||
println("✓ Chains verified ($(round(total_size_gb, digits=2)) GB total)")
|
||||
|
||||
else
|
||||
# Legacy mode - create new run directory and copy chains
|
||||
temp_csv_dir = run_dir_or_temp
|
||||
|
||||
timestamp = Dates.format(Dates.now(), "yyyy-mm-dd_HH-MM-SS")
|
||||
run_id = "run_$(timestamp)"
|
||||
run_dir = joinpath("outputs", "model_outputs", "latest", run_id)
|
||||
chains_dir = joinpath(run_dir, "chains")
|
||||
data_dir = joinpath(run_dir, "data")
|
||||
|
||||
println("Run ID: $run_id")
|
||||
println("Output directory: $run_dir")
|
||||
|
||||
# Create directory structure
|
||||
mkpath(chains_dir)
|
||||
|
||||
# STEP 1: Copy CSV chain files
|
||||
println("\n" * "=" ^ 70)
|
||||
println("STEP 1: Copying MCMC chain CSV files")
|
||||
println("=" ^ 70)
|
||||
println("Source: $temp_csv_dir")
|
||||
println("Destination: $chains_dir")
|
||||
|
||||
csv_files = filter(f -> endswith(f, ".csv"), readdir(temp_csv_dir))
|
||||
chain_files = filter(f -> contains(f, "chain"), csv_files)
|
||||
|
||||
if isempty(chain_files)
|
||||
error("No chain CSV files found in $temp_csv_dir")
|
||||
end
|
||||
|
||||
println("Found $(length(chain_files)) chain files")
|
||||
|
||||
total_size_gb = 0.0
|
||||
for (i, csv_file) in enumerate(sort(chain_files))
|
||||
src_path = joinpath(temp_csv_dir, csv_file)
|
||||
|
||||
# Rename to standard format: chain_1.csv, chain_2.csv, etc.
|
||||
dest_filename = "chain_$i.csv"
|
||||
dest_path = joinpath(chains_dir, dest_filename)
|
||||
|
||||
src_size = filesize(src_path)
|
||||
size_gb = src_size / (1024^3)
|
||||
total_size_gb += size_gb
|
||||
|
||||
println(" Copying $csv_file → $dest_filename ($(round(size_gb, digits=2)) GB)")
|
||||
cp(src_path, dest_path, force=true)
|
||||
|
||||
# Verify copy with size check
|
||||
dest_size = filesize(dest_path)
|
||||
if dest_size != src_size
|
||||
error("CRITICAL: Size mismatch for $dest_filename! Source: $src_size, Dest: $dest_size")
|
||||
end
|
||||
end
|
||||
|
||||
println("✓ All chains copied and verified ($(round(total_size_gb, digits=2)) GB total)")
|
||||
end
|
||||
|
||||
# Create data directory
|
||||
mkpath(data_dir)
|
||||
|
||||
# STEP 2: Save data CSVs
|
||||
println("\n" * "=" ^ 70)
|
||||
println("STEP 2: Saving original data CSVs")
|
||||
println("=" ^ 70)
|
||||
|
||||
for (name, df) in original_data
|
||||
if isa(df, DataFrame)
|
||||
csv_path = joinpath(data_dir, "$(name).csv")
|
||||
println(" Saving $(name).csv ($(nrow(df)) rows)")
|
||||
CSV.write(csv_path, df)
|
||||
end
|
||||
end
|
||||
|
||||
println("✓ Data CSVs saved")
|
||||
|
||||
# STEP 3: Save Stan data dictionary as JSON
|
||||
println("\n" * "=" ^ 70)
|
||||
println("STEP 3: Saving Stan data dictionary")
|
||||
println("=" ^ 70)
|
||||
|
||||
# Convert data_dict to JSON-serializable format
|
||||
stan_data_json = Dict{String, Any}()
|
||||
for (k, v) in data_dict
|
||||
try
|
||||
# Only save simple types (numbers, arrays of numbers)
|
||||
if isa(v, Number) || isa(v, AbstractArray{<:Number})
|
||||
stan_data_json[k] = v
|
||||
elseif isa(v, AbstractArray)
|
||||
# Try to convert, skip if fails
|
||||
try
|
||||
stan_data_json[k] = collect(v)
|
||||
catch
|
||||
println(" Skipping $k (complex type)")
|
||||
end
|
||||
end
|
||||
catch e
|
||||
println(" Warning: Could not serialize $k: $e")
|
||||
end
|
||||
end
|
||||
|
||||
stan_data_path = joinpath(data_dir, "stan_data.json")
|
||||
open(stan_data_path, "w") do f
|
||||
JSON.print(f, stan_data_json, 2)
|
||||
end
|
||||
println("✓ Stan data saved to stan_data.json")
|
||||
|
||||
# Count chain files for metadata
|
||||
chain_files_final = filter(f -> endswith(f, ".csv") && contains(f, "chain"), readdir(chains_dir))
|
||||
num_chains = length(chain_files_final)
|
||||
|
||||
# Recalculate total_size_gb if in chains_already_saved mode
|
||||
if chains_already_saved
|
||||
total_size_gb = 0.0
|
||||
for chain_file in chain_files_final
|
||||
chain_path = joinpath(chains_dir, chain_file)
|
||||
total_size_gb += filesize(chain_path) / (1024^3)
|
||||
end
|
||||
end
|
||||
|
||||
# STEP 4: Save metadata
|
||||
println("\n" * "=" ^ 70)
|
||||
println("STEP 4: Saving metadata")
|
||||
println("=" ^ 70)
|
||||
|
||||
# Add run info to metadata
|
||||
metadata["run_id"] = run_id
|
||||
metadata["timestamp"] = timestamp
|
||||
metadata["files"] = Dict(
|
||||
"chains" => ["chains/chain_$i.csv" for i in 1:num_chains],
|
||||
"data" => readdir(data_dir),
|
||||
"chain_size_gb" => num_chains > 0 ? round(total_size_gb / num_chains, digits=2) : 0.0,
|
||||
"total_size_gb" => round(total_size_gb, digits=2)
|
||||
)
|
||||
|
||||
metadata_path = joinpath(run_dir, "metadata.json")
|
||||
open(metadata_path, "w") do f
|
||||
JSON.print(f, metadata, 2)
|
||||
end
|
||||
println("✓ Metadata saved to metadata.json")
|
||||
|
||||
# STEP 5: Generate README
|
||||
println("\n" * "=" ^ 70)
|
||||
println("STEP 5: Generating README")
|
||||
println("=" ^ 70)
|
||||
|
||||
readme_path = joinpath(run_dir, "README.txt")
|
||||
generate_readme(readme_path, run_id, metadata, num_chains, total_size_gb)
|
||||
println("✓ README generated")
|
||||
|
||||
# STEP 6: Final verification
|
||||
println("\n" * "=" ^ 70)
|
||||
println("STEP 6: Verification")
|
||||
println("=" ^ 70)
|
||||
|
||||
# Verify all chain files exist and are readable
|
||||
all_good = true
|
||||
verified_files = Dict{String, Dict{String, Any}}()
|
||||
|
||||
for i in 1:num_chains
|
||||
chain_path = joinpath(chains_dir, "chain_$i.csv")
|
||||
if !isfile(chain_path)
|
||||
println(" ✗ Missing: chain_$i.csv")
|
||||
all_good = false
|
||||
else
|
||||
# Quick read test and size check
|
||||
try
|
||||
CSV.File(chain_path; limit=1)
|
||||
file_size_gb = filesize(chain_path) / (1024^3)
|
||||
verified_files["chain_$i.csv"] = Dict(
|
||||
"path" => chain_path,
|
||||
"size_gb" => file_size_gb,
|
||||
"verified" => true
|
||||
)
|
||||
println(" ✓ chain_$i.csv verified ($(round(file_size_gb, digits=2)) GB)")
|
||||
catch e
|
||||
println(" ✗ Cannot read chain_$i.csv: $e")
|
||||
all_good = false
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if !all_good
|
||||
error("Verification failed - some files are missing or corrupted")
|
||||
end
|
||||
|
||||
println("\n" * "=" ^ 70)
|
||||
println("✓ MODEL SAVED SUCCESSFULLY")
|
||||
println("=" ^ 70)
|
||||
println("Run directory: $run_dir")
|
||||
println("Total size: $(round(total_size_gb, digits=2)) GB")
|
||||
println("Status: All files verified and ready")
|
||||
println("=" ^ 70)
|
||||
|
||||
# Return verification details for cleanup
|
||||
return (
|
||||
run_dir = run_dir,
|
||||
verified_files = verified_files,
|
||||
total_size_gb = total_size_gb,
|
||||
verification_passed = all_good,
|
||||
num_chains = num_chains
|
||||
)
|
||||
end
|
||||
|
||||
function generate_readme(
|
||||
filepath::String,
|
||||
run_id::String,
|
||||
metadata::Dict,
|
||||
num_chains::Int,
|
||||
total_size_gb::Float64
|
||||
)
|
||||
"""Generate human-readable README file"""
|
||||
|
||||
open(filepath, "w") do f
|
||||
write(f, "=" ^ 78 * "\n")
|
||||
write(f, "4D LATENT TRAIT MODEL - MODEL RUN RESULTS\n")
|
||||
write(f, "=" ^ 78 * "\n\n")
|
||||
|
||||
write(f, "Run ID: $run_id\n")
|
||||
write(f, "Model: $(get(metadata, "model_file", "unknown"))\n")
|
||||
write(f, "Date: $(Dates.format(Dates.now(), "yyyy-mm-dd HH:MM:SS"))\n")
|
||||
write(f, "Status: $(get(metadata, "convergence_status", "unknown"))\n\n")
|
||||
|
||||
write(f, "=" ^ 78 * "\n")
|
||||
write(f, "DIRECTORY CONTENTS\n")
|
||||
write(f, "=" ^ 78 * "\n\n")
|
||||
|
||||
write(f, "chains/\n")
|
||||
for i in 1:num_chains
|
||||
write(f, " ├── chain_$i.csv\n")
|
||||
end
|
||||
write(f, " Total: $(get(metadata, "num_chains", num_chains)) chains × " *
|
||||
"$(get(metadata, "num_samples", "?")) samples\n")
|
||||
write(f, " Size: $(round(total_size_gb, digits=2)) GB\n\n")
|
||||
|
||||
write(f, "data/\n")
|
||||
write(f, " ├── text_data.csv\n")
|
||||
write(f, " ├── expert_dim.csv\n")
|
||||
write(f, " ├── expert_lr.csv\n")
|
||||
write(f, " ├── segment_year_map.csv (V10)\n")
|
||||
write(f, " ├── segment_info.csv (V10)\n")
|
||||
write(f, " └── stan_data.json\n\n")
|
||||
|
||||
write(f, "=" ^ 78 * "\n")
|
||||
write(f, "MODEL CONFIGURATION\n")
|
||||
write(f, "=" ^ 78 * "\n\n")
|
||||
|
||||
write(f, "Chains: $(get(metadata, "num_chains", "?"))\n")
|
||||
write(f, "Warmup: $(get(metadata, "num_warmup", "?"))\n")
|
||||
write(f, "Samples: $(get(metadata, "num_samples", "?"))\n")
|
||||
write(f, "Adapt delta: $(get(metadata, "adapt_delta", "?"))\n")
|
||||
write(f, "Max depth: $(get(metadata, "max_depth", "?"))\n\n")
|
||||
|
||||
write(f, "Dimensions: $(join(get(metadata, "dimensions", ["?"]), ", "))\n\n")
|
||||
|
||||
write(f, "=" ^ 78 * "\n")
|
||||
write(f, "HOW TO USE THESE RESULTS\n")
|
||||
write(f, "=" ^ 78 * "\n\n")
|
||||
|
||||
write(f, "To extract party positions:\n\n")
|
||||
write(f, " julia 02_post_estimation.jl\n\n")
|
||||
|
||||
write(f, "This will read the CSV files and generate party_positions_[timestamp].csv\n")
|
||||
write(f, "with uncertainty estimates (SE, credible intervals).\n\n")
|
||||
|
||||
write(f, "=" ^ 78 * "\n")
|
||||
write(f, "Generated: $(Dates.format(Dates.now(), "yyyy-mm-dd HH:MM:SS"))\n")
|
||||
write(f, "=" ^ 78 * "\n")
|
||||
end
|
||||
end
|
||||
|
||||
"""
|
||||
Wrapper for robust_save_model_csv that handles the stanmodel tuple from run_4dim_stan_model.
|
||||
|
||||
The model execution now saves chains BEFORE returning, so this function:
|
||||
1. Verifies chains are already saved in final_run_dir
|
||||
2. Adds metadata and data files
|
||||
3. Returns the output path
|
||||
|
||||
Arguments:
|
||||
- stanmodel_tuple: Named tuple from run_4dim_stan_model (contains .stanmodel, .final_run_dir, etc.)
|
||||
- model_data: Dict with data_dict, original data, and metadata
|
||||
- output_dir: Base output directory (ignored - uses stanmodel_tuple.final_run_dir)
|
||||
- compress: Ignored (CSV-first architecture)
|
||||
- keep_local_backups: Ignored (chains already saved)
|
||||
"""
|
||||
function robust_save_model(
|
||||
stanmodel_tuple,
|
||||
model_data::Dict,
|
||||
output_dir::String;
|
||||
compress::Bool=true,
|
||||
keep_local_backups::Int=2
|
||||
)
|
||||
# Extract the final run directory from the stanmodel tuple
|
||||
if !hasproperty(stanmodel_tuple, :final_run_dir)
|
||||
error("stanmodel_tuple missing :final_run_dir - chains may not be saved!")
|
||||
end
|
||||
|
||||
final_run_dir = stanmodel_tuple.final_run_dir
|
||||
|
||||
# Prepare original data for saving
|
||||
original_data = Dict{String, Any}()
|
||||
if haskey(model_data, "manifesto")
|
||||
original_data["text_data"] = model_data["manifesto"]
|
||||
end
|
||||
if haskey(model_data, "expert_dim")
|
||||
original_data["expert_dim"] = model_data["expert_dim"]
|
||||
end
|
||||
if haskey(model_data, "expert_lr")
|
||||
original_data["expert_lr"] = model_data["expert_lr"]
|
||||
end
|
||||
# V10: Save segment_year_map and segment_info for post-estimation
|
||||
if haskey(model_data, "segment_year")
|
||||
original_data["segment_year_map"] = model_data["segment_year"]
|
||||
end
|
||||
if haskey(model_data, "segment_info")
|
||||
original_data["segment_info"] = model_data["segment_info"]
|
||||
end
|
||||
# V9 fallback: Save party_year_map for post-estimation (includes interpolated years)
|
||||
if haskey(model_data, "party_year")
|
||||
original_data["party_year_map"] = model_data["party_year"]
|
||||
end
|
||||
|
||||
# Prepare metadata
|
||||
metadata = Dict{String, Any}()
|
||||
if haskey(model_data, "model_info")
|
||||
for (k, v) in model_data["model_info"]
|
||||
metadata[string(k)] = v
|
||||
end
|
||||
end
|
||||
|
||||
# Get data_dict
|
||||
data_dict = get(model_data, "data_dict", Dict{String, Any}())
|
||||
|
||||
# Call the CSV-first save with chains_already_saved=true
|
||||
result = robust_save_model_csv(
|
||||
final_run_dir,
|
||||
data_dict,
|
||||
original_data,
|
||||
metadata;
|
||||
chains_already_saved=true
|
||||
)
|
||||
|
||||
return result.run_dir
|
||||
end
|
||||
|
||||
end # module
|
||||
@@ -0,0 +1,150 @@
|
||||
#!/usr/bin/env julia
|
||||
#############################################################################
|
||||
## performance_monitoring.jl
|
||||
## Post-run performance diagnostics for Stan sampling jobs
|
||||
#############################################################################
|
||||
|
||||
using StanSample
|
||||
using Statistics: mean, median
|
||||
using Dates
|
||||
|
||||
function _safe_column(df, candidates::Vector{String})
|
||||
for candidate in candidates
|
||||
sym = Symbol(candidate)
|
||||
if sym in names(df)
|
||||
return df[!, sym]
|
||||
elseif candidate in names(df)
|
||||
return df[!, candidate]
|
||||
end
|
||||
end
|
||||
return nothing
|
||||
end
|
||||
|
||||
function _clean_values(vec)
|
||||
cleaned = Float64[]
|
||||
for v in vec
|
||||
if v isa Missing || v === nothing
|
||||
continue
|
||||
end
|
||||
try
|
||||
value = Float64(v)
|
||||
isfinite(value) && push!(cleaned, value)
|
||||
catch
|
||||
continue
|
||||
end
|
||||
end
|
||||
return cleaned
|
||||
end
|
||||
|
||||
function _summarize_vector(vec)
|
||||
if vec === nothing
|
||||
return Dict{String,Any}("available" => false)
|
||||
end
|
||||
cleaned = _clean_values(vec)
|
||||
if isempty(cleaned)
|
||||
return Dict{String,Any}("available" => false)
|
||||
end
|
||||
return Dict{String,Any}(
|
||||
"available" => true,
|
||||
"count" => length(cleaned),
|
||||
"mean" => mean(cleaned),
|
||||
"median" => median(cleaned),
|
||||
"min" => minimum(cleaned),
|
||||
"max" => maximum(cleaned)
|
||||
)
|
||||
end
|
||||
|
||||
function monitor_sampling_performance!(stanmodel;
|
||||
run_metrics::Union{Nothing,Dict{String,Any}}=nothing,
|
||||
metrics_path::Union{Nothing,String}=nothing,
|
||||
csv_paths::Union{Nothing,Vector{String}}=nothing,
|
||||
aggregate_metrics::Union{Nothing,Dict{String,Any}}=nothing,
|
||||
max_depth::Union{Nothing,Int}=nothing)
|
||||
|
||||
csv_paths === nothing && (csv_paths = discover_stan_csvs([stanmodel.tmpdir]))
|
||||
aggregate_metrics === nothing && begin
|
||||
_, aggregate_metrics = collect_run_metrics(csv_paths; max_depth=max_depth)
|
||||
end
|
||||
|
||||
summary_df = nothing
|
||||
try
|
||||
summary_df = read_summary(stanmodel)
|
||||
catch e
|
||||
println("Warning: could not read Stan summary: $e")
|
||||
end
|
||||
|
||||
performance = Dict{String,Any}(
|
||||
"generated_at" => Dates.format(Dates.now(), "yyyy-mm-ddTHH:MM:SS"),
|
||||
"csv_paths" => csv_paths
|
||||
)
|
||||
|
||||
if summary_df !== nothing
|
||||
performance["ess_bulk"] = _summarize_vector(_safe_column(summary_df, ["ess_bulk", "ess"]))
|
||||
performance["ess_tail"] = _summarize_vector(_safe_column(summary_df, ["ess_tail"]))
|
||||
performance["ess_per_sec"] = _summarize_vector(_safe_column(summary_df, ["ess_per_sec", "n_eff/s"]))
|
||||
performance["r_hat"] = _summarize_vector(_safe_column(summary_df, ["r_hat"]))
|
||||
|
||||
if haskey(performance["r_hat"], "available") && performance["r_hat"]["available"]
|
||||
performance["r_hat"]["max"] = maximum(_clean_values(_safe_column(summary_df, ["r_hat"])))
|
||||
end
|
||||
|
||||
performance["parameters_considered"] = size(summary_df, 1)
|
||||
else
|
||||
performance["ess_bulk"] = Dict{String,Any}("available" => false)
|
||||
performance["ess_tail"] = Dict{String,Any}("available" => false)
|
||||
performance["ess_per_sec"] = Dict{String,Any}("available" => false)
|
||||
performance["r_hat"] = Dict{String,Any}("available" => false)
|
||||
performance["parameters_considered"] = 0
|
||||
end
|
||||
|
||||
divergences = get(aggregate_metrics, "divergences", 0)
|
||||
total_samples = stanmodel.num_samples * stanmodel.num_chains
|
||||
divergence_rate = total_samples > 0 ? divergences / total_samples : nothing
|
||||
|
||||
performance["divergences"] = Dict{String,Any}(
|
||||
"count" => divergences,
|
||||
"rate" => divergence_rate,
|
||||
"total_draws" => total_samples
|
||||
)
|
||||
|
||||
performance["leapfrog"] = Dict{String,Any}(
|
||||
"mean" => get(aggregate_metrics, "mean_leapfrog", nothing)
|
||||
)
|
||||
|
||||
performance["step_size"] = Dict{String,Any}(
|
||||
"mean" => get(aggregate_metrics, "mean_step_size", nothing)
|
||||
)
|
||||
|
||||
sampling_seconds = get(aggregate_metrics, "sampling_seconds", nothing)
|
||||
if sampling_seconds !== nothing && sampling_seconds > 0
|
||||
performance["throughput"] = Dict{String,Any}(
|
||||
"samples_per_second" => (total_samples / sampling_seconds),
|
||||
"seconds_sampling" => sampling_seconds
|
||||
)
|
||||
else
|
||||
performance["throughput"] = Dict{String,Any}(
|
||||
"samples_per_second" => nothing,
|
||||
"seconds_sampling" => sampling_seconds
|
||||
)
|
||||
end
|
||||
|
||||
if run_metrics !== nothing
|
||||
run_metrics["performance"] = performance
|
||||
if metrics_path !== nothing
|
||||
safe_write_json(metrics_path, run_metrics)
|
||||
end
|
||||
elseif metrics_path !== nothing
|
||||
temp_metrics = Dict{String,Any}("performance" => performance)
|
||||
safe_write_json(metrics_path, temp_metrics)
|
||||
end
|
||||
|
||||
println("\nPERFORMANCE SUMMARY")
|
||||
println(" ESS bulk (mean): $(get(performance["ess_bulk"], "mean", "n/a"))")
|
||||
println(" ESS/sec (mean): $(get(performance["ess_per_sec"], "mean", "n/a"))")
|
||||
println(" Divergences: $(divergences)")
|
||||
println(" Divergence rate: $(divergence_rate === nothing ? "n/a" : round(divergence_rate, digits=6))")
|
||||
println(" Mean leapfrog steps: $(get(performance["leapfrog"], "mean", "n/a"))")
|
||||
|
||||
return performance
|
||||
end
|
||||
|
||||
@@ -0,0 +1,450 @@
|
||||
#!/usr/bin/env julia
|
||||
#############################################################################
|
||||
## validate_construct.jl
|
||||
## Construct validity: Party family ordering and temporal stability
|
||||
##
|
||||
## Following Claassen (2019), this script validates:
|
||||
## 1. Party family ordering: Do family means follow theoretically expected orderings?
|
||||
## 2. Temporal stability: Flag parties with implausible position changes
|
||||
##
|
||||
## Uses ParlGov party family classifications (Döring & Manow 2024) via PartyFacts IDs.
|
||||
#############################################################################
|
||||
|
||||
using CSV, DataFrames, Statistics, StatsBase, Dates, Printf
|
||||
|
||||
# Family code → display name mapping
|
||||
const FAMILY_DISPLAY_NAMES = Dict(
|
||||
"com" => "Communist/Far Left",
|
||||
"eco" => "Green/Ecological",
|
||||
"soc" => "Social Democratic",
|
||||
"lib" => "Liberal",
|
||||
"chr" => "Christian Democratic",
|
||||
"con" => "Conservative",
|
||||
"right" => "Radical Right"
|
||||
)
|
||||
|
||||
# Substantive families (drop Specialist, Other, Agrarian — heterogeneous or ambiguous)
|
||||
const SUBSTANTIVE_FAMILIES = Set(["com", "eco", "soc", "lib", "chr", "con", "right"])
|
||||
|
||||
# Expected orderings (theoretically motivated)
|
||||
# Economic: Communist < Social Democratic < Green < Christian Democratic < Conservative
|
||||
# (5-family core — Liberal position is ambiguous cross-nationally)
|
||||
const EXPECTED_ECONOMIC_ORDER = ["com", "soc", "eco", "chr", "con"]
|
||||
|
||||
# Cultural: Green < Liberal < Social Democratic < Christian Democratic < Conservative < Radical Right
|
||||
const EXPECTED_GALTAN_ORDER = ["eco", "lib", "soc", "chr", "con", "right"]
|
||||
|
||||
function load_model_output(base_dir::String=".")
|
||||
"""Load the most recent 2D model party positions output"""
|
||||
|
||||
position_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
|
||||
!endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(base_dir))
|
||||
legacy_files = filter(f -> startswith(f, "party_positions_v1_") && endswith(f, ".csv"), readdir(base_dir))
|
||||
append!(position_files, legacy_files)
|
||||
|
||||
if !isempty(position_files)
|
||||
latest = sort(position_files)[end]
|
||||
println("Loading model output: $latest")
|
||||
return CSV.read(joinpath(base_dir, latest), DataFrame), latest
|
||||
end
|
||||
|
||||
# Check output estimations directory
|
||||
est_dir = joinpath(base_dir, "outputs", "estimations", "latest")
|
||||
if isdir(est_dir)
|
||||
est_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
|
||||
!endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(est_dir))
|
||||
if !isempty(est_files)
|
||||
latest = sort(est_files)[end]
|
||||
println("Loading model output: outputs/estimations/latest/$latest")
|
||||
return CSV.read(joinpath(est_dir, latest), DataFrame), latest
|
||||
end
|
||||
end
|
||||
|
||||
error("No party_positions_*.csv found. Run 02_post_estimation.jl first.")
|
||||
end
|
||||
|
||||
function validate_party_families(model::DataFrame)
|
||||
"""Check whether party family means follow theoretically expected orderings"""
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("PARTY FAMILY ORDERING VALIDATION")
|
||||
println("="^60)
|
||||
println("\nUsing ParlGov family classifications (Döring & Manow 2024)")
|
||||
println()
|
||||
|
||||
party_col = hasproperty(model, :party_id) ? :party_id : :party
|
||||
|
||||
# Load party families
|
||||
families_df = CSV.read("data/party_families.csv", DataFrame)
|
||||
|
||||
# Join to model output
|
||||
model_with_families = innerjoin(model, families_df, on=party_col => :partyfacts_id)
|
||||
|
||||
# Filter to substantive families
|
||||
filter!(r -> r.family in SUBSTANTIVE_FAMILIES, model_with_families)
|
||||
|
||||
n_parties = length(unique(model_with_families[!, party_col]))
|
||||
n_obs = nrow(model_with_families)
|
||||
println(" Matched $n_parties parties ($n_obs party-years) across $(length(SUBSTANTIVE_FAMILIES)) families")
|
||||
println()
|
||||
|
||||
# Compute family means
|
||||
family_stats = combine(groupby(model_with_families, :family)) do df
|
||||
DataFrame(
|
||||
n_parties = length(unique(df[!, party_col])),
|
||||
n_obs = nrow(df),
|
||||
mean_economic = mean(df.economic_lr),
|
||||
sd_economic = std(df.economic_lr),
|
||||
mean_galtan = mean(df.galtan),
|
||||
sd_galtan = std(df.galtan)
|
||||
)
|
||||
end
|
||||
|
||||
# Add display names
|
||||
family_stats.family_name = [get(FAMILY_DISPLAY_NAMES, f, f) for f in family_stats.family]
|
||||
|
||||
# Sort by economic mean for display
|
||||
sort!(family_stats, :mean_economic)
|
||||
|
||||
# Print table
|
||||
@printf(" %-22s %7s %7s %10s %10s %10s %10s\n",
|
||||
"Family", "Parties", "Obs", "Econ Mean", "Econ SD", "Cult Mean", "Cult SD")
|
||||
println(" " * "-"^76)
|
||||
|
||||
for row in eachrow(family_stats)
|
||||
@printf(" %-22s %7d %7d %10.3f %10.3f %10.3f %10.3f\n",
|
||||
row.family_name, row.n_parties, row.n_obs,
|
||||
row.mean_economic, row.sd_economic, row.mean_galtan, row.sd_galtan)
|
||||
end
|
||||
|
||||
# Compute Spearman rank correlations for expected orderings
|
||||
println()
|
||||
|
||||
# Economic ordering
|
||||
econ_lookup = Dict(row.family => row.mean_economic for row in eachrow(family_stats))
|
||||
econ_observed = [econ_lookup[f] for f in EXPECTED_ECONOMIC_ORDER if haskey(econ_lookup, f)]
|
||||
econ_expected_ranks = collect(1:length(econ_observed))
|
||||
econ_observed_ranks = ordinalrank(econ_observed)
|
||||
rho_econ = corspearman(Float64.(econ_expected_ranks), Float64.(econ_observed_ranks))
|
||||
|
||||
println(@sprintf(" Economic ordering (5-family core): Spearman ρ = %.3f", rho_econ))
|
||||
econ_families_used = [f for f in EXPECTED_ECONOMIC_ORDER if haskey(econ_lookup, f)]
|
||||
econ_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in econ_families_used]
|
||||
println(" Expected: ", join(econ_names, " < "))
|
||||
observed_econ_order = econ_families_used[sortperm(econ_observed)]
|
||||
observed_econ_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in observed_econ_order]
|
||||
println(" Observed: ", join(observed_econ_names, " < "))
|
||||
|
||||
# Cultural ordering
|
||||
galtan_lookup = Dict(row.family => row.mean_galtan for row in eachrow(family_stats))
|
||||
galtan_observed = [galtan_lookup[f] for f in EXPECTED_GALTAN_ORDER if haskey(galtan_lookup, f)]
|
||||
galtan_expected_ranks = collect(1:length(galtan_observed))
|
||||
galtan_observed_ranks = ordinalrank(galtan_observed)
|
||||
rho_galtan = corspearman(Float64.(galtan_expected_ranks), Float64.(galtan_observed_ranks))
|
||||
|
||||
println(@sprintf(" Cultural ordering (6-family): Spearman ρ = %.3f", rho_galtan))
|
||||
galtan_families_used = [f for f in EXPECTED_GALTAN_ORDER if haskey(galtan_lookup, f)]
|
||||
galtan_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in galtan_families_used]
|
||||
println(" Expected: ", join(galtan_names, " < "))
|
||||
observed_galtan_order = galtan_families_used[sortperm(galtan_observed)]
|
||||
observed_galtan_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in observed_galtan_order]
|
||||
println(" Observed: ", join(observed_galtan_names, " < "))
|
||||
|
||||
println()
|
||||
println("-"^60)
|
||||
if rho_econ >= 0.9 && rho_galtan >= 0.8
|
||||
println("EXCELLENT: Family means follow expected orderings on both dimensions")
|
||||
elseif rho_econ >= 0.7 && rho_galtan >= 0.7
|
||||
println("GOOD: Family means broadly follow expected orderings")
|
||||
else
|
||||
println("CONCERN: Review family ordering results")
|
||||
end
|
||||
|
||||
return family_stats, rho_econ, rho_galtan
|
||||
end
|
||||
|
||||
function validate_temporal_stability(model::DataFrame)
|
||||
"""Check for implausible year-to-year position changes"""
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("TEMPORAL STABILITY VALIDATION")
|
||||
println("="^60)
|
||||
println("\nFlagging parties with >0.10 change per year")
|
||||
println()
|
||||
|
||||
party_col = hasproperty(model, :party_id) ? :party_id : :party
|
||||
|
||||
# Compute year-to-year changes within each party
|
||||
sort!(model, [party_col, :year])
|
||||
|
||||
unstable_parties = []
|
||||
|
||||
for party_df in groupby(model, party_col)
|
||||
if nrow(party_df) < 2
|
||||
continue
|
||||
end
|
||||
|
||||
party_id = party_df[1, party_col]
|
||||
country = party_df[1, :country]
|
||||
|
||||
# Compute differences
|
||||
for dim in [:economic_lr, :galtan]
|
||||
vals = party_df[!, dim]
|
||||
years = party_df.year
|
||||
|
||||
for i in 2:length(vals)
|
||||
diff = abs(vals[i] - vals[i-1])
|
||||
year_gap = years[i] - years[i-1]
|
||||
|
||||
# Normalize by year gap (handle multi-year gaps)
|
||||
annual_change = diff / max(year_gap, 1)
|
||||
|
||||
if annual_change > 0.10
|
||||
push!(unstable_parties, (
|
||||
party_id = party_id,
|
||||
country = country,
|
||||
dimension = string(dim),
|
||||
year_from = years[i-1],
|
||||
year_to = years[i],
|
||||
val_from = vals[i-1],
|
||||
val_to = vals[i],
|
||||
change = diff,
|
||||
annual_change = annual_change
|
||||
))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
if isempty(unstable_parties)
|
||||
println(" No parties with >0.10 annual change found")
|
||||
println(" EXCELLENT: Positions are temporally stable")
|
||||
return DataFrame()
|
||||
end
|
||||
|
||||
unstable_df = DataFrame(unstable_parties)
|
||||
sort!(unstable_df, :annual_change, rev=true)
|
||||
|
||||
println(" Found $(nrow(unstable_df)) instances of rapid change:")
|
||||
println()
|
||||
@printf(" %-8s %-8s %-12s %-10s %-10s %8s\n",
|
||||
"Party", "Country", "Dimension", "Years", "Change", "Annual")
|
||||
println(" " * "-"^60)
|
||||
|
||||
for row in eachrow(unstable_df[1:min(20, nrow(unstable_df)), :])
|
||||
@printf(" %-8d %-8s %-12s %d->%d %8.3f %8.3f\n",
|
||||
row.party_id, row.country, row.dimension,
|
||||
row.year_from, row.year_to, row.change, row.annual_change)
|
||||
end
|
||||
|
||||
if nrow(unstable_df) > 20
|
||||
println(" ... and $(nrow(unstable_df) - 20) more")
|
||||
end
|
||||
|
||||
println()
|
||||
println("-"^60)
|
||||
n_parties = length(unique(unstable_df.party_id))
|
||||
n_total = length(unique(model[!, party_col]))
|
||||
println(@sprintf("Unstable parties: %d/%d (%.1f%%)", n_parties, n_total, 100*n_parties/n_total))
|
||||
|
||||
return unstable_df
|
||||
end
|
||||
|
||||
function validate_position_distributions(model::DataFrame)
|
||||
"""Check overall distribution of positions makes sense"""
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("POSITION DISTRIBUTION VALIDATION")
|
||||
println("="^60)
|
||||
println("\nSummary statistics for model estimates")
|
||||
println()
|
||||
|
||||
for dim in [:economic_lr, :galtan]
|
||||
if !hasproperty(model, dim)
|
||||
continue
|
||||
end
|
||||
|
||||
vals = model[!, dim]
|
||||
println("$dim:")
|
||||
println(@sprintf(" Mean: %.3f (should be ~0.50)", mean(vals)))
|
||||
println(@sprintf(" Median: %.3f (should be ~0.50)", median(vals)))
|
||||
println(@sprintf(" SD: %.3f (should be ~0.15)", std(vals)))
|
||||
println(@sprintf(" Min: %.3f", minimum(vals)))
|
||||
println(@sprintf(" Max: %.3f", maximum(vals)))
|
||||
println(@sprintf(" Q25: %.3f", quantile(vals, 0.25)))
|
||||
println(@sprintf(" Q75: %.3f", quantile(vals, 0.75)))
|
||||
println()
|
||||
end
|
||||
|
||||
# Check for extreme values
|
||||
println("Extreme positions (< 0.10 or > 0.90):")
|
||||
|
||||
party_col = hasproperty(model, :party_id) ? :party_id : :party
|
||||
|
||||
for dim in [:economic_lr, :galtan]
|
||||
if !hasproperty(model, dim)
|
||||
continue
|
||||
end
|
||||
|
||||
extreme = filter(row -> row[dim] < 0.10 || row[dim] > 0.90, model)
|
||||
n_extreme = nrow(extreme)
|
||||
pct_extreme = 100 * n_extreme / nrow(model)
|
||||
|
||||
println(@sprintf(" %s: %d (%.1f%%)", dim, n_extreme, pct_extreme))
|
||||
|
||||
if n_extreme > 0 && n_extreme <= 10
|
||||
for row in eachrow(extreme[1:min(5, nrow(extreme)), :])
|
||||
println(@sprintf(" Party %d (%s) %d: %.3f",
|
||||
row[party_col], row.country, row.year, row[dim]))
|
||||
end
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
function validate_country_patterns(model::DataFrame)
|
||||
"""Check country-level patterns make sense"""
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("COUNTRY-LEVEL VALIDATION")
|
||||
println("="^60)
|
||||
println("\nMean positions by country (should vary but not wildly)")
|
||||
println()
|
||||
|
||||
country_stats = combine(groupby(model, :country)) do df
|
||||
DataFrame(
|
||||
n_parties = length(unique(hasproperty(df, :party_id) ? df.party_id : df.party)),
|
||||
n_obs = nrow(df),
|
||||
mean_econ = mean(df.economic_lr),
|
||||
mean_galtan = mean(df.galtan),
|
||||
sd_econ = std(df.economic_lr),
|
||||
sd_galtan = std(df.galtan)
|
||||
)
|
||||
end
|
||||
|
||||
sort!(country_stats, :n_obs, rev=true)
|
||||
|
||||
@printf("%-4s %6s %6s %8s %8s %8s %8s\n",
|
||||
"CC", "Parties", "N", "Econ", "SD", "Galtan", "SD")
|
||||
println("-"^60)
|
||||
|
||||
for row in eachrow(country_stats[1:min(20, nrow(country_stats)), :])
|
||||
@printf("%-4s %6d %6d %8.3f %8.3f %8.3f %8.3f\n",
|
||||
row.country, row.n_parties, row.n_obs,
|
||||
row.mean_econ, row.sd_econ, row.mean_galtan, row.sd_galtan)
|
||||
end
|
||||
|
||||
# Flag countries with unusual patterns
|
||||
println("\nCountries with unusual patterns:")
|
||||
unusual = filter(row -> row.mean_econ < 0.35 || row.mean_econ > 0.65 ||
|
||||
row.mean_galtan < 0.35 || row.mean_galtan > 0.65, country_stats)
|
||||
|
||||
if nrow(unusual) == 0
|
||||
println(" None - all countries have balanced party systems")
|
||||
else
|
||||
for row in eachrow(unusual)
|
||||
issues = String[]
|
||||
if row.mean_econ < 0.35
|
||||
push!(issues, "left-skewed economy")
|
||||
elseif row.mean_econ > 0.65
|
||||
push!(issues, "right-skewed economy")
|
||||
end
|
||||
if row.mean_galtan < 0.35
|
||||
push!(issues, "cultural-skewed")
|
||||
elseif row.mean_galtan > 0.65
|
||||
push!(issues, "TAN-skewed")
|
||||
end
|
||||
println(" $(row.country): $(join(issues, ", "))")
|
||||
end
|
||||
end
|
||||
|
||||
return country_stats
|
||||
end
|
||||
|
||||
function save_construct_results(families::DataFrame, unstable::DataFrame,
|
||||
countries::DataFrame, output_dir::String="outputs/checks")
|
||||
"""Save construct validation results"""
|
||||
|
||||
if !isdir(output_dir)
|
||||
mkpath(output_dir)
|
||||
end
|
||||
|
||||
timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")
|
||||
|
||||
if nrow(families) > 0
|
||||
families_file = joinpath(output_dir, "construct_families_$timestamp.csv")
|
||||
CSV.write(families_file, families)
|
||||
println("\nSaved: $families_file")
|
||||
end
|
||||
|
||||
if nrow(unstable) > 0
|
||||
unstable_file = joinpath(output_dir, "construct_unstable_$timestamp.csv")
|
||||
CSV.write(unstable_file, unstable)
|
||||
println("Saved: $unstable_file")
|
||||
end
|
||||
|
||||
if nrow(countries) > 0
|
||||
country_file = joinpath(output_dir, "construct_countries_$timestamp.csv")
|
||||
CSV.write(country_file, countries)
|
||||
println("Saved: $country_file")
|
||||
end
|
||||
end
|
||||
|
||||
|
||||
# Main execution
|
||||
function main()
|
||||
println("="^60)
|
||||
println("CONSTRUCT VALIDITY: Face Validity Checks")
|
||||
println("="^60)
|
||||
println("Checking if model estimates match expectations")
|
||||
println()
|
||||
|
||||
# Load data
|
||||
model, model_file = load_model_output()
|
||||
|
||||
println("\nModel output:")
|
||||
println(" Rows: $(nrow(model))")
|
||||
println(" Columns: $(names(model))")
|
||||
|
||||
# Run validations
|
||||
families, rho_econ, rho_galtan = validate_party_families(model)
|
||||
unstable = validate_temporal_stability(model)
|
||||
validate_position_distributions(model)
|
||||
countries = validate_country_patterns(model)
|
||||
|
||||
# Save results
|
||||
save_construct_results(families, unstable, countries)
|
||||
|
||||
# Summary
|
||||
println("\n" * "="^60)
|
||||
println("CONSTRUCT VALIDITY SUMMARY")
|
||||
println("="^60)
|
||||
|
||||
println(@sprintf(" Party family ordering:"))
|
||||
println(@sprintf(" Economic (5-family): Spearman ρ = %.3f", rho_econ))
|
||||
println(@sprintf(" Cultural (6-family): Spearman ρ = %.3f", rho_galtan))
|
||||
|
||||
n_unstable = nrow(unstable) > 0 ? length(unique(unstable.party_id)) : 0
|
||||
party_col = hasproperty(model, :party_id) ? :party_id : :party
|
||||
n_total = length(unique(model[!, party_col]))
|
||||
println(@sprintf(" Temporal stability: %d/%d parties stable (>0.10/yr threshold)",
|
||||
n_total - n_unstable, n_total))
|
||||
|
||||
if rho_econ >= 0.9 && rho_galtan >= 0.8 && n_unstable < 0.1 * n_total
|
||||
println("\n EXCELLENT: Model has good construct validity")
|
||||
elseif rho_econ >= 0.7 && rho_galtan >= 0.7
|
||||
println("\n GOOD: Model has reasonable construct validity")
|
||||
else
|
||||
println("\n CONCERN: Review family ordering results")
|
||||
end
|
||||
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("VALIDATION COMPLETE")
|
||||
println("="^60)
|
||||
|
||||
return (families=families, unstable=unstable, countries=countries)
|
||||
end
|
||||
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
main()
|
||||
end
|
||||
@@ -0,0 +1,533 @@
|
||||
#!/usr/bin/env julia
|
||||
#############################################################################
|
||||
## validate_convergent.jl
|
||||
## Convergent validity: Compare model estimates to external expert surveys
|
||||
##
|
||||
## Following Claassen (2019), this script computes:
|
||||
## - Pearson/Spearman correlations between model and expert estimates
|
||||
## - Fisher z-transformation for correlation confidence intervals
|
||||
## - Mean Absolute Error (MAE) and Root Mean Square Error (RMSE)
|
||||
## - Breakdown by survey project and decade
|
||||
##
|
||||
## Target: r > 0.8 with CHES (Claassen achieved 0.50-0.57)
|
||||
#############################################################################
|
||||
|
||||
using CSV, DataFrames, Statistics, StatsBase, Dates, Printf, JSON
|
||||
|
||||
# Fisher z-transformation for correlation confidence intervals
|
||||
fisher_z(r) = 0.5 * log((1 + r) / (1 - r))
|
||||
fisher_z_inv(z) = (exp(2z) - 1) / (exp(2z) + 1)
|
||||
|
||||
function correlation_ci(r, n; alpha=0.05)
|
||||
"""Calculate confidence interval for correlation using Fisher z-transformation"""
|
||||
if n < 4
|
||||
return (lower=NaN, upper=NaN)
|
||||
end
|
||||
z = fisher_z(r)
|
||||
se = 1 / sqrt(n - 3)
|
||||
z_crit = 1.96 # For 95% CI
|
||||
z_lower = z - z_crit * se
|
||||
z_upper = z + z_crit * se
|
||||
return (lower=fisher_z_inv(z_lower), upper=fisher_z_inv(z_upper))
|
||||
end
|
||||
|
||||
function load_model_output(base_dir::String=".")
|
||||
"""Load the most recent 2D model party positions output"""
|
||||
|
||||
# First check for post_estimation output in root (current name, with legacy fallback)
|
||||
position_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
|
||||
!endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(base_dir))
|
||||
legacy_files = filter(f -> startswith(f, "party_positions_v1_") && endswith(f, ".csv"), readdir(base_dir))
|
||||
append!(position_files, legacy_files)
|
||||
|
||||
if !isempty(position_files)
|
||||
latest = sort(position_files)[end]
|
||||
println("Loading model output: $latest")
|
||||
return CSV.read(joinpath(base_dir, latest), DataFrame), latest
|
||||
end
|
||||
|
||||
# Check current pipeline output directory, with legacy estimations/ fallback
|
||||
for (label, est_dir) in [
|
||||
("outputs/estimations/latest", joinpath(base_dir, "outputs", "estimations", "latest")),
|
||||
("estimations", joinpath(base_dir, "estimations")),
|
||||
]
|
||||
if isdir(est_dir)
|
||||
est_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
|
||||
!endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(est_dir))
|
||||
if !isempty(est_files)
|
||||
latest = sort(est_files)[end]
|
||||
println("Loading model output: $label/$latest")
|
||||
return CSV.read(joinpath(est_dir, latest), DataFrame), latest
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
error("No party_positions_*.csv found. Run 02_post_estimation.jl first.")
|
||||
end
|
||||
|
||||
function load_expert_data(base_dir::String=".")
|
||||
"""Load expert survey data files"""
|
||||
|
||||
data_dir = isfile(joinpath(base_dir, "expert.csv")) ? base_dir : joinpath(base_dir, "data")
|
||||
|
||||
# Load dimension-specific expert data
|
||||
expert_file = joinpath(data_dir, "expert.csv")
|
||||
if !isfile(expert_file)
|
||||
error("expert.csv not found in $base_dir or $(joinpath(base_dir, "data"))")
|
||||
end
|
||||
|
||||
println("Loading expert.csv...")
|
||||
expert = CSV.read(expert_file, DataFrame)
|
||||
println(" Rows: $(nrow(expert))")
|
||||
println(" Variables: $(unique(expert.var))")
|
||||
|
||||
# Load L-R data
|
||||
lr_file = joinpath(data_dir, "lr_data.csv")
|
||||
if !isfile(lr_file)
|
||||
error("lr_data.csv not found in $base_dir or $(joinpath(base_dir, "data"))")
|
||||
end
|
||||
|
||||
println("Loading lr_data.csv...")
|
||||
lr_data = CSV.read(lr_file, DataFrame)
|
||||
println(" Rows: $(nrow(lr_data))")
|
||||
println(" Variables: $(unique(lr_data.var))")
|
||||
|
||||
return expert, lr_data
|
||||
end
|
||||
|
||||
function validate_economic_lr(model::DataFrame, expert::DataFrame)
|
||||
"""Validate economic_lr against CHES/V-Party/POPPA/GPS lrecon"""
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("CONVERGENT VALIDITY: economic_lr")
|
||||
println("="^60)
|
||||
|
||||
# Filter expert data for economic dimension
|
||||
econ_vars = filter(v -> startswith(v, "lrecon_"), unique(expert.var))
|
||||
econ_expert = filter(row -> row.var in econ_vars, expert)
|
||||
|
||||
println("\nExpert data variables: $(econ_vars)")
|
||||
println("Expert observations: $(nrow(econ_expert))")
|
||||
|
||||
# Merge with model output
|
||||
# Model has party_id column (from segment-based), expert has party column
|
||||
if hasproperty(model, :party_id)
|
||||
model_merge = select(model, :party_id => :party, :year, :economic_lr, :economic_lr_se)
|
||||
else
|
||||
model_merge = select(model, :party, :year, :economic_lr, :economic_lr_se)
|
||||
end
|
||||
|
||||
merged = innerjoin(econ_expert, model_merge, on=[:party, :year])
|
||||
|
||||
println("Merged observations: $(nrow(merged))")
|
||||
|
||||
if nrow(merged) < 10
|
||||
println("WARNING: Too few observations for meaningful validation")
|
||||
return nothing
|
||||
end
|
||||
|
||||
# Compute overall correlation
|
||||
r_pearson = cor(merged.val, merged.economic_lr)
|
||||
r_spearman = corspearman(merged.val, merged.economic_lr)
|
||||
mae = mean(abs.(merged.val .- merged.economic_lr))
|
||||
rmse = sqrt(mean((merged.val .- merged.economic_lr).^2))
|
||||
|
||||
ci = correlation_ci(r_pearson, nrow(merged))
|
||||
|
||||
println("\n--- Overall Statistics ---")
|
||||
println(@sprintf(" Pearson r: %.4f [%.4f, %.4f]", r_pearson, ci.lower, ci.upper))
|
||||
println(@sprintf(" Spearman r: %.4f", r_spearman))
|
||||
println(@sprintf(" MAE: %.4f", mae))
|
||||
println(@sprintf(" RMSE: %.4f", rmse))
|
||||
println(@sprintf(" N: %d", nrow(merged)))
|
||||
|
||||
# Breakdown by project
|
||||
println("\n--- By Project ---")
|
||||
by_project = combine(groupby(merged, :project)) do df
|
||||
n = nrow(df)
|
||||
if n < 3
|
||||
return DataFrame(n=n, r_pearson=NaN, mae=NaN)
|
||||
end
|
||||
DataFrame(
|
||||
n = n,
|
||||
r_pearson = cor(df.val, df.economic_lr),
|
||||
r_spearman = corspearman(df.val, df.economic_lr),
|
||||
mae = mean(abs.(df.val .- df.economic_lr)),
|
||||
rmse = sqrt(mean((df.val .- df.economic_lr).^2))
|
||||
)
|
||||
end
|
||||
|
||||
for row in eachrow(sort(by_project, :n, rev=true))
|
||||
if !isnan(row.r_pearson)
|
||||
println(@sprintf(" %-10s: r=%.3f, MAE=%.3f, n=%d",
|
||||
row.project, row.r_pearson, row.mae, row.n))
|
||||
end
|
||||
end
|
||||
|
||||
# Breakdown by decade
|
||||
println("\n--- By Decade ---")
|
||||
merged.decade = div.(merged.year, 10) .* 10
|
||||
by_decade = combine(groupby(merged, :decade)) do df
|
||||
n = nrow(df)
|
||||
if n < 3
|
||||
return DataFrame(n=n, r_pearson=NaN, mae=NaN)
|
||||
end
|
||||
DataFrame(
|
||||
n = n,
|
||||
r_pearson = cor(df.val, df.economic_lr),
|
||||
mae = mean(abs.(df.val .- df.economic_lr))
|
||||
)
|
||||
end
|
||||
|
||||
for row in eachrow(sort(by_decade, :decade))
|
||||
if !isnan(row.r_pearson)
|
||||
println(@sprintf(" %ds: r=%.3f, MAE=%.3f, n=%d",
|
||||
row.decade, row.r_pearson, row.mae, row.n))
|
||||
end
|
||||
end
|
||||
|
||||
return (
|
||||
dimension = "economic_lr",
|
||||
r_pearson = r_pearson,
|
||||
r_spearman = r_spearman,
|
||||
ci_lower = ci.lower,
|
||||
ci_upper = ci.upper,
|
||||
mae = mae,
|
||||
rmse = rmse,
|
||||
n = nrow(merged),
|
||||
by_project = by_project,
|
||||
by_decade = by_decade
|
||||
)
|
||||
end
|
||||
|
||||
function validate_galtan(model::DataFrame, expert::DataFrame)
|
||||
"""Validate galtan against CHES galtan and V-Party/GPS libcons"""
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("CONVERGENT VALIDITY: galtan")
|
||||
println("="^60)
|
||||
|
||||
# Filter expert data for GAL-TAN dimension
|
||||
galtan_vars = filter(v -> occursin("galtan", v) || occursin("libcon", v), unique(expert.var))
|
||||
galtan_expert = filter(row -> row.var in galtan_vars, expert)
|
||||
|
||||
println("\nExpert data variables: $(galtan_vars)")
|
||||
println("Expert observations: $(nrow(galtan_expert))")
|
||||
|
||||
# Merge with model output
|
||||
if hasproperty(model, :party_id)
|
||||
model_merge = select(model, :party_id => :party, :year, :galtan, :galtan_se)
|
||||
else
|
||||
model_merge = select(model, :party, :year, :galtan, :galtan_se)
|
||||
end
|
||||
|
||||
merged = innerjoin(galtan_expert, model_merge, on=[:party, :year])
|
||||
|
||||
println("Merged observations: $(nrow(merged))")
|
||||
|
||||
if nrow(merged) < 10
|
||||
println("WARNING: Too few observations for meaningful validation")
|
||||
return nothing
|
||||
end
|
||||
|
||||
# Compute overall correlation
|
||||
r_pearson = cor(merged.val, merged.galtan)
|
||||
r_spearman = corspearman(merged.val, merged.galtan)
|
||||
mae = mean(abs.(merged.val .- merged.galtan))
|
||||
rmse = sqrt(mean((merged.val .- merged.galtan).^2))
|
||||
|
||||
ci = correlation_ci(r_pearson, nrow(merged))
|
||||
|
||||
println("\n--- Overall Statistics ---")
|
||||
println(@sprintf(" Pearson r: %.4f [%.4f, %.4f]", r_pearson, ci.lower, ci.upper))
|
||||
println(@sprintf(" Spearman r: %.4f", r_spearman))
|
||||
println(@sprintf(" MAE: %.4f", mae))
|
||||
println(@sprintf(" RMSE: %.4f", rmse))
|
||||
println(@sprintf(" N: %d", nrow(merged)))
|
||||
|
||||
# Breakdown by project
|
||||
println("\n--- By Project ---")
|
||||
by_project = combine(groupby(merged, :project)) do df
|
||||
n = nrow(df)
|
||||
if n < 3
|
||||
return DataFrame(n=n, r_pearson=NaN, mae=NaN)
|
||||
end
|
||||
DataFrame(
|
||||
n = n,
|
||||
r_pearson = cor(df.val, df.galtan),
|
||||
r_spearman = corspearman(df.val, df.galtan),
|
||||
mae = mean(abs.(df.val .- df.galtan)),
|
||||
rmse = sqrt(mean((df.val .- df.galtan).^2))
|
||||
)
|
||||
end
|
||||
|
||||
for row in eachrow(sort(by_project, :n, rev=true))
|
||||
if !isnan(row.r_pearson)
|
||||
println(@sprintf(" %-10s: r=%.3f, MAE=%.3f, n=%d",
|
||||
row.project, row.r_pearson, row.mae, row.n))
|
||||
end
|
||||
end
|
||||
|
||||
# Breakdown by decade
|
||||
println("\n--- By Decade ---")
|
||||
merged.decade = div.(merged.year, 10) .* 10
|
||||
by_decade = combine(groupby(merged, :decade)) do df
|
||||
n = nrow(df)
|
||||
if n < 3
|
||||
return DataFrame(n=n, r_pearson=NaN, mae=NaN)
|
||||
end
|
||||
DataFrame(
|
||||
n = n,
|
||||
r_pearson = cor(df.val, df.galtan),
|
||||
mae = mean(abs.(df.val .- df.galtan))
|
||||
)
|
||||
end
|
||||
|
||||
for row in eachrow(sort(by_decade, :decade))
|
||||
if !isnan(row.r_pearson)
|
||||
println(@sprintf(" %ds: r=%.3f, MAE=%.3f, n=%d",
|
||||
row.decade, row.r_pearson, row.mae, row.n))
|
||||
end
|
||||
end
|
||||
|
||||
return (
|
||||
dimension = "galtan",
|
||||
r_pearson = r_pearson,
|
||||
r_spearman = r_spearman,
|
||||
ci_lower = ci.lower,
|
||||
ci_upper = ci.upper,
|
||||
mae = mae,
|
||||
rmse = rmse,
|
||||
n = nrow(merged),
|
||||
by_project = by_project,
|
||||
by_decade = by_decade
|
||||
)
|
||||
end
|
||||
|
||||
function validate_discriminant(model::DataFrame, expert::DataFrame)
|
||||
"""Compute cross-dimension correlations for discriminant validity (Campbell & Fiske 1959 MTMM)"""
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("DISCRIMINANT VALIDITY: Cross-dimension correlations")
|
||||
println("="^60)
|
||||
println("\nCampbell & Fiske (1959) MTMM framework:")
|
||||
println(" Convergent: same dimension, different method → HIGH")
|
||||
println(" Discriminant: different dimension, different method → LOW")
|
||||
println()
|
||||
|
||||
# Get party column
|
||||
if hasproperty(model, :party_id)
|
||||
model_econ = select(model, :party_id => :party, :year, :economic_lr)
|
||||
model_gal = select(model, :party_id => :party, :year, :galtan)
|
||||
else
|
||||
model_econ = select(model, :party, :year, :economic_lr)
|
||||
model_gal = select(model, :party, :year, :galtan)
|
||||
end
|
||||
|
||||
results = []
|
||||
|
||||
# 1. Expert economic vs Model economic (convergent - already computed, include for matrix)
|
||||
econ_vars = filter(v -> startswith(v, "lrecon_"), unique(expert.var))
|
||||
econ_expert = filter(row -> row.var in econ_vars, expert)
|
||||
merged_ee = innerjoin(econ_expert, model_econ, on=[:party, :year])
|
||||
if nrow(merged_ee) >= 10
|
||||
r = cor(merged_ee.val, merged_ee.economic_lr)
|
||||
push!(results, (model_dim="economic_lr", expert_dim="economic",
|
||||
r_pearson=r, r_spearman=corspearman(merged_ee.val, merged_ee.economic_lr),
|
||||
n=nrow(merged_ee), type="convergent"))
|
||||
@printf(" Model Economic × Expert Economic: r = %.3f (convergent, n=%d)\n", r, nrow(merged_ee))
|
||||
end
|
||||
|
||||
# 2. Expert economic vs Model galtan (discriminant)
|
||||
merged_eg = innerjoin(econ_expert, model_gal, on=[:party, :year])
|
||||
if nrow(merged_eg) >= 10
|
||||
r = cor(merged_eg.val, merged_eg.galtan)
|
||||
push!(results, (model_dim="galtan", expert_dim="economic",
|
||||
r_pearson=r, r_spearman=corspearman(merged_eg.val, merged_eg.galtan),
|
||||
n=nrow(merged_eg), type="discriminant"))
|
||||
@printf(" Model GAL-TAN × Expert Economic: r = %.3f (discriminant, n=%d)\n", r, nrow(merged_eg))
|
||||
end
|
||||
|
||||
# 3. Expert galtan vs Model galtan (convergent - already computed, include for matrix)
|
||||
galtan_vars = filter(v -> occursin("galtan", v) || occursin("libcon", v), unique(expert.var))
|
||||
galtan_expert = filter(row -> row.var in galtan_vars, expert)
|
||||
merged_gg = innerjoin(galtan_expert, model_gal, on=[:party, :year])
|
||||
if nrow(merged_gg) >= 10
|
||||
r = cor(merged_gg.val, merged_gg.galtan)
|
||||
push!(results, (model_dim="galtan", expert_dim="galtan",
|
||||
r_pearson=r, r_spearman=corspearman(merged_gg.val, merged_gg.galtan),
|
||||
n=nrow(merged_gg), type="convergent"))
|
||||
@printf(" Model GAL-TAN × Expert GAL-TAN: r = %.3f (convergent, n=%d)\n", r, nrow(merged_gg))
|
||||
end
|
||||
|
||||
# 4. Expert galtan vs Model economic (discriminant)
|
||||
merged_ge = innerjoin(galtan_expert, model_econ, on=[:party, :year])
|
||||
if nrow(merged_ge) >= 10
|
||||
r = cor(merged_ge.val, merged_ge.economic_lr)
|
||||
push!(results, (model_dim="economic_lr", expert_dim="galtan",
|
||||
r_pearson=r, r_spearman=corspearman(merged_ge.val, merged_ge.economic_lr),
|
||||
n=nrow(merged_ge), type="discriminant"))
|
||||
@printf(" Model Economic × Expert GAL-TAN: r = %.3f (discriminant, n=%d)\n", r, nrow(merged_ge))
|
||||
end
|
||||
|
||||
println()
|
||||
println("MTMM Matrix:")
|
||||
println(" Expert Economic Expert GAL-TAN")
|
||||
for r in results
|
||||
if r.model_dim == "economic_lr" && r.expert_dim == "economic"
|
||||
@printf(" Model Economic: %.3f ", r.r_pearson)
|
||||
end
|
||||
end
|
||||
for r in results
|
||||
if r.model_dim == "economic_lr" && r.expert_dim == "galtan"
|
||||
@printf("%.3f\n", r.r_pearson)
|
||||
end
|
||||
end
|
||||
for r in results
|
||||
if r.model_dim == "galtan" && r.expert_dim == "economic"
|
||||
@printf(" Model GAL-TAN: %.3f ", r.r_pearson)
|
||||
end
|
||||
end
|
||||
for r in results
|
||||
if r.model_dim == "galtan" && r.expert_dim == "galtan"
|
||||
@printf("%.3f\n", r.r_pearson)
|
||||
end
|
||||
end
|
||||
|
||||
return DataFrame(results)
|
||||
end
|
||||
|
||||
function save_validation_results(results::Vector, output_dir::String="validation";
|
||||
discriminant::Union{DataFrame, Nothing}=nothing)
|
||||
"""Save validation results to CSV files"""
|
||||
|
||||
if !isdir(output_dir)
|
||||
mkpath(output_dir)
|
||||
end
|
||||
|
||||
timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")
|
||||
|
||||
# Summary table
|
||||
summary_rows = []
|
||||
for r in results
|
||||
if r !== nothing
|
||||
push!(summary_rows, (
|
||||
dimension = r.dimension,
|
||||
r_pearson = r.r_pearson,
|
||||
r_spearman = r.r_spearman,
|
||||
ci_lower = r.ci_lower,
|
||||
ci_upper = r.ci_upper,
|
||||
mae = r.mae,
|
||||
rmse = r.rmse,
|
||||
n = r.n
|
||||
))
|
||||
end
|
||||
end
|
||||
|
||||
if !isempty(summary_rows)
|
||||
summary_df = DataFrame(summary_rows)
|
||||
summary_file = joinpath(output_dir, "convergent_summary_$timestamp.csv")
|
||||
CSV.write(summary_file, summary_df)
|
||||
println("\nSaved: $summary_file")
|
||||
end
|
||||
|
||||
# By-project tables
|
||||
for r in results
|
||||
if r !== nothing && hasproperty(r, :by_project) && r.by_project !== nothing
|
||||
project_file = joinpath(output_dir, "convergent_$(r.dimension)_by_project_$timestamp.csv")
|
||||
CSV.write(project_file, r.by_project)
|
||||
println("Saved: $project_file")
|
||||
end
|
||||
end
|
||||
|
||||
# By-decade tables
|
||||
for r in results
|
||||
if r !== nothing && hasproperty(r, :by_decade) && r.by_decade !== nothing
|
||||
decade_file = joinpath(output_dir, "convergent_$(r.dimension)_by_decade_$timestamp.csv")
|
||||
CSV.write(decade_file, r.by_decade)
|
||||
println("Saved: $decade_file")
|
||||
end
|
||||
end
|
||||
|
||||
# Discriminant validity table
|
||||
if discriminant !== nothing && nrow(discriminant) > 0
|
||||
disc_file = joinpath(output_dir, "discriminant_summary_$timestamp.csv")
|
||||
CSV.write(disc_file, discriminant)
|
||||
println("Saved: $disc_file")
|
||||
end
|
||||
|
||||
return summary_rows
|
||||
end
|
||||
|
||||
function print_claassen_comparison(results::Vector)
|
||||
"""Print comparison with Claassen (2019) benchmarks"""
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("COMPARISON WITH CLAASSEN (2019) BENCHMARKS")
|
||||
println("="^60)
|
||||
|
||||
println("\nClaassen's results (mood estimates vs survey data):")
|
||||
println(" Pearson r: 0.50-0.57")
|
||||
println(" MAE: ~0.06 (6 pp on 0-1 scale)")
|
||||
println()
|
||||
|
||||
println("Our target (party positions, should be HIGHER than mood):")
|
||||
println(" Pearson r > 0.80 with expert surveys")
|
||||
println(" MAE < 0.15 (reasonable measurement error)")
|
||||
println()
|
||||
|
||||
println("-"^60)
|
||||
@printf("%-15s %8s %8s %8s %8s\n", "Dimension", "r", "Target", "MAE", "Status")
|
||||
println("-"^60)
|
||||
|
||||
for r in results
|
||||
if r !== nothing
|
||||
status = r.r_pearson > 0.80 ? "PASS" : (r.r_pearson > 0.70 ? "OK" : "LOW")
|
||||
@printf("%-15s %8.3f %8s %8.3f %8s\n",
|
||||
r.dimension, r.r_pearson, "> 0.80", r.mae, status)
|
||||
end
|
||||
end
|
||||
println("-"^60)
|
||||
end
|
||||
|
||||
# Main execution
|
||||
function main()
|
||||
println("="^60)
|
||||
println("CONVERGENT VALIDITY: Model vs Expert Surveys")
|
||||
println("="^60)
|
||||
println("Following Claassen (2019) validation framework")
|
||||
println()
|
||||
|
||||
# Load data
|
||||
model, model_file = load_model_output()
|
||||
expert, _ = load_expert_data()
|
||||
|
||||
println("\nModel output:")
|
||||
println(" Rows: $(nrow(model))")
|
||||
println(" Columns: $(names(model))")
|
||||
|
||||
# Run validations
|
||||
results = []
|
||||
|
||||
push!(results, validate_economic_lr(model, expert))
|
||||
push!(results, validate_galtan(model, expert))
|
||||
|
||||
# Run discriminant validity
|
||||
discriminant = validate_discriminant(model, expert)
|
||||
|
||||
# Save results
|
||||
save_validation_results(results; discriminant=discriminant)
|
||||
|
||||
# Print Claassen comparison
|
||||
print_claassen_comparison(results)
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("VALIDATION COMPLETE")
|
||||
println("="^60)
|
||||
|
||||
return results
|
||||
end
|
||||
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
main()
|
||||
end
|
||||
@@ -0,0 +1,375 @@
|
||||
#!/usr/bin/env julia
|
||||
#############################################################################
|
||||
## validate_external.jl
|
||||
## Out-of-sample validation via held-out expert observations
|
||||
##
|
||||
## Design:
|
||||
## - Text data stays 100% intact (same parties, segments, indices)
|
||||
## - 20% of expert/LR observations held out (stratified by source)
|
||||
## - Expert-only parties (no text data) are never held out
|
||||
## - Model trains on 80% expert + 100% text
|
||||
## - Held-out expert ratings compared to model predictions
|
||||
##
|
||||
## Usage:
|
||||
## julia scripts/validate_external.jl prepare
|
||||
## julia 01_run_model.jl --data-dir validation/external_split/
|
||||
## julia 02_post_estimation.jl # on training run
|
||||
## julia scripts/validate_external.jl compute <model_positions.csv>
|
||||
#############################################################################
|
||||
|
||||
using CSV, DataFrames, Statistics, Random, Dates, Printf
|
||||
|
||||
const HOLDOUT_FRAC = 0.20
|
||||
const SEED = 42
|
||||
|
||||
# =========================================================================
|
||||
# STEP 1: Prepare train/test split
|
||||
# =========================================================================
|
||||
|
||||
function prepare_holdout_data(base_dir::String=".")
|
||||
println("="^70)
|
||||
println("PREPARING OUT-OF-SAMPLE VALIDATION SPLIT")
|
||||
println("="^70)
|
||||
println()
|
||||
|
||||
# Load data
|
||||
text_data = CSV.read(joinpath(base_dir, "text_data.csv"), DataFrame)
|
||||
expert = CSV.read(joinpath(base_dir, "expert.csv"), DataFrame)
|
||||
lr_data = CSV.read(joinpath(base_dir, "lr_data.csv"), DataFrame)
|
||||
|
||||
println("Full dataset:")
|
||||
println(" text_data: $(nrow(text_data)) rows, $(length(unique(text_data.party))) parties")
|
||||
println(" expert: $(nrow(expert)) rows, $(length(unique(expert.party))) parties")
|
||||
println(" lr_data: $(nrow(lr_data)) rows, $(length(unique(lr_data.party))) parties")
|
||||
|
||||
# Identify expert-only parties (no text data) — these are NEVER held out
|
||||
text_parties = Set(unique(text_data.party))
|
||||
expert_only_parties = Set(p for p in unique(vcat(expert.party, lr_data.party))
|
||||
if !(p in text_parties))
|
||||
|
||||
println()
|
||||
println("Expert-only parties (protected from holdout): $(length(expert_only_parties))")
|
||||
|
||||
# Split expert data: stratified by source variable
|
||||
# Safeguard: ensure each party keeps at least one observation in training
|
||||
Random.seed!(SEED)
|
||||
expert.row_id = 1:nrow(expert)
|
||||
expert.is_holdout = falses(nrow(expert))
|
||||
|
||||
for var_group in groupby(expert, :var)
|
||||
var_name = first(var_group.var)
|
||||
eligible = findall(row -> !(row.party in expert_only_parties), eachrow(var_group))
|
||||
|
||||
n_holdout = round(Int, length(eligible) * HOLDOUT_FRAC)
|
||||
holdout_candidates = shuffle(eligible)
|
||||
|
||||
# Track per-party counts to ensure at least 1 stays in training
|
||||
party_train_count = Dict{Int, Int}()
|
||||
for idx in eligible
|
||||
p = var_group.party[idx]
|
||||
party_train_count[p] = get(party_train_count, p, 0) + 1
|
||||
end
|
||||
|
||||
n_held = 0
|
||||
for idx in holdout_candidates
|
||||
n_held >= n_holdout && break
|
||||
p = var_group.party[idx]
|
||||
if party_train_count[p] > 1 # keep at least 1 in training
|
||||
row_id = var_group.row_id[idx]
|
||||
expert.is_holdout[row_id] = true
|
||||
party_train_count[p] -= 1
|
||||
n_held += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Split LR data: stratified by source variable (same safeguard)
|
||||
lr_data.row_id = 1:nrow(lr_data)
|
||||
lr_data.is_holdout = falses(nrow(lr_data))
|
||||
|
||||
for var_group in groupby(lr_data, :var)
|
||||
var_name = first(var_group.var)
|
||||
eligible = findall(row -> !(row.party in expert_only_parties), eachrow(var_group))
|
||||
|
||||
n_holdout = round(Int, length(eligible) * HOLDOUT_FRAC)
|
||||
holdout_candidates = shuffle(eligible)
|
||||
|
||||
party_train_count = Dict{Int, Int}()
|
||||
for idx in eligible
|
||||
p = var_group.party[idx]
|
||||
party_train_count[p] = get(party_train_count, p, 0) + 1
|
||||
end
|
||||
|
||||
n_held = 0
|
||||
for idx in holdout_candidates
|
||||
n_held >= n_holdout && break
|
||||
p = var_group.party[idx]
|
||||
if party_train_count[p] > 1
|
||||
row_id = var_group.row_id[idx]
|
||||
lr_data.is_holdout[row_id] = true
|
||||
party_train_count[p] -= 1
|
||||
n_held += 1
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# Create train/test splits
|
||||
expert_train = expert[.!expert.is_holdout, Not([:row_id, :is_holdout])]
|
||||
expert_test = expert[expert.is_holdout, Not([:row_id, :is_holdout])]
|
||||
lr_train = lr_data[.!lr_data.is_holdout, Not([:row_id, :is_holdout])]
|
||||
lr_test = lr_data[lr_data.is_holdout, Not([:row_id, :is_holdout])]
|
||||
|
||||
# Report split
|
||||
println()
|
||||
println("Split summary ($(round(100*HOLDOUT_FRAC))% holdout):")
|
||||
println(" expert train: $(nrow(expert_train)) rows ($(round(100*nrow(expert_train)/nrow(expert), digits=1))%)")
|
||||
println(" expert test: $(nrow(expert_test)) rows ($(round(100*nrow(expert_test)/nrow(expert), digits=1))%)")
|
||||
println(" lr train: $(nrow(lr_train)) rows ($(round(100*nrow(lr_train)/nrow(lr_data), digits=1))%)")
|
||||
println(" lr test: $(nrow(lr_test)) rows ($(round(100*nrow(lr_test)/nrow(lr_data), digits=1))%)")
|
||||
|
||||
# Report per-source breakdown
|
||||
println()
|
||||
println("Per-source breakdown (expert):")
|
||||
for var_name in sort(unique(expert.var))
|
||||
n_full = count(expert.var .== var_name)
|
||||
n_test = count(expert_test.var .== var_name)
|
||||
println(" $var_name: $(n_full - n_test) train / $n_test test")
|
||||
end
|
||||
println()
|
||||
println("Per-source breakdown (LR):")
|
||||
for var_name in sort(unique(lr_data.var))
|
||||
n_full = count(lr_data.var .== var_name)
|
||||
n_test = count(lr_test.var .== var_name)
|
||||
println(" $var_name: $(n_full - n_test) train / $n_test test")
|
||||
end
|
||||
|
||||
# Verify: training set has same parties as full set
|
||||
train_parties_expert = Set(unique(expert_train.party))
|
||||
train_parties_lr = Set(unique(lr_train.party))
|
||||
full_parties_expert = Set(unique(expert.party))
|
||||
full_parties_lr = Set(unique(lr_data.party))
|
||||
|
||||
lost_expert = setdiff(full_parties_expert, train_parties_expert)
|
||||
lost_lr = setdiff(full_parties_lr, train_parties_lr)
|
||||
|
||||
println()
|
||||
if isempty(lost_expert) && isempty(lost_lr)
|
||||
println("✓ No parties lost from training set")
|
||||
else
|
||||
println("⚠ Parties lost from expert training: $(length(lost_expert))")
|
||||
println("⚠ Parties lost from LR training: $(length(lost_lr))")
|
||||
end
|
||||
|
||||
# Save to output directory
|
||||
# Files use standard names so 01_run_model.jl can load with --data-dir
|
||||
output_dir = joinpath(base_dir, "validation", "external_split")
|
||||
mkpath(output_dir)
|
||||
|
||||
# Training files (standard names for model loading)
|
||||
CSV.write(joinpath(output_dir, "text_data.csv"), text_data) # UNCHANGED
|
||||
CSV.write(joinpath(output_dir, "expert.csv"), expert_train)
|
||||
CSV.write(joinpath(output_dir, "lr_data.csv"), lr_train)
|
||||
|
||||
# Test files (for compute step)
|
||||
CSV.write(joinpath(output_dir, "expert_test.csv"), expert_test)
|
||||
CSV.write(joinpath(output_dir, "lr_data_test.csv"), lr_test)
|
||||
|
||||
# Copy union mapping (needed by model)
|
||||
if isdir(joinpath(base_dir, "data"))
|
||||
mkpath(joinpath(output_dir, "data"))
|
||||
cp(joinpath(base_dir, "data", "union_mapping.csv"),
|
||||
joinpath(output_dir, "data", "union_mapping.csv"), force=true)
|
||||
end
|
||||
|
||||
println()
|
||||
println("Files saved to: $output_dir")
|
||||
println(" text_data.csv — IDENTICAL to original ($(nrow(text_data)) rows)")
|
||||
println(" expert.csv — training only ($(nrow(expert_train)) rows)")
|
||||
println(" lr_data.csv — training only ($(nrow(lr_train)) rows)")
|
||||
println(" expert_test.csv — held-out ($(nrow(expert_test)) rows)")
|
||||
println(" lr_data_test.csv — held-out ($(nrow(lr_test)) rows)")
|
||||
|
||||
return output_dir
|
||||
end
|
||||
|
||||
# =========================================================================
|
||||
# STEP 2: Compute held-out validation metrics
|
||||
# =========================================================================
|
||||
|
||||
function compute_holdout_metrics(model_file::String, test_dir::String)
|
||||
println()
|
||||
println("="^70)
|
||||
println("COMPUTING HELD-OUT VALIDATION METRICS")
|
||||
println("="^70)
|
||||
|
||||
# Load model output
|
||||
model = CSV.read(model_file, DataFrame)
|
||||
party_col = hasproperty(model, :party_id) ? :party_id : :party
|
||||
println("Model output: $(nrow(model)) party-years")
|
||||
|
||||
# Load test data
|
||||
expert_test = CSV.read(joinpath(test_dir, "expert_test.csv"), DataFrame)
|
||||
lr_test = CSV.read(joinpath(test_dir, "lr_data_test.csv"), DataFrame)
|
||||
println("Held-out expert: $(nrow(expert_test)) observations")
|
||||
println("Held-out LR: $(nrow(lr_test)) observations")
|
||||
|
||||
# Build lookup: (party, year) → model estimates
|
||||
model_lookup = Dict{Tuple{Int,Int}, NamedTuple}()
|
||||
for row in eachrow(model)
|
||||
key = (row[party_col], row.year)
|
||||
model_lookup[key] = (
|
||||
economic_lr = row.economic_lr,
|
||||
galtan = row.galtan,
|
||||
economic_lr_se = hasproperty(row, :economic_lr_se) ? row.economic_lr_se : missing,
|
||||
galtan_se = hasproperty(row, :galtan_se) ? row.galtan_se : missing,
|
||||
economic_lr_q025 = hasproperty(row, :economic_lr_q025) ? row.economic_lr_q025 : missing,
|
||||
economic_lr_q975 = hasproperty(row, :economic_lr_q975) ? row.economic_lr_q975 : missing,
|
||||
galtan_q025 = hasproperty(row, :galtan_q025) ? row.galtan_q025 : missing,
|
||||
galtan_q975 = hasproperty(row, :galtan_q975) ? row.galtan_q975 : missing,
|
||||
)
|
||||
end
|
||||
|
||||
# Map expert variables to dimensions
|
||||
econ_vars = Set(["lrecon_ches", "lrecon_poppa", "lrecon_gps", "lrecon_vparty", "welf_vparty"])
|
||||
galtan_vars = Set(["galtan_ches", "libcon_gps", "immig_vparty", "lgbt_vparty",
|
||||
"culsup_vparty", "relig_vparty", "gender_vparty"])
|
||||
|
||||
# Process expert test observations
|
||||
results = NamedTuple[]
|
||||
|
||||
for row in eachrow(expert_test)
|
||||
key = (row.party, row.year)
|
||||
haskey(model_lookup, key) || continue
|
||||
m = model_lookup[key]
|
||||
|
||||
if row.var in econ_vars
|
||||
dim = "economic_lr"
|
||||
model_val = m.economic_lr
|
||||
model_q025 = m.economic_lr_q025
|
||||
model_q975 = m.economic_lr_q975
|
||||
elseif row.var in galtan_vars
|
||||
dim = "galtan"
|
||||
model_val = m.galtan
|
||||
model_q025 = m.galtan_q025
|
||||
model_q975 = m.galtan_q975
|
||||
else
|
||||
continue
|
||||
end
|
||||
|
||||
covered = !ismissing(model_q025) && !ismissing(model_q975) &&
|
||||
row.val >= model_q025 && row.val <= model_q975
|
||||
|
||||
push!(results, (
|
||||
party = row.party,
|
||||
country = row.country,
|
||||
year = row.year,
|
||||
var = row.var,
|
||||
dimension = dim,
|
||||
expert_val = row.val,
|
||||
model_val = model_val,
|
||||
error = row.val - model_val,
|
||||
abs_error = abs(row.val - model_val),
|
||||
covered_95 = ismissing(model_q025) ? missing : covered,
|
||||
))
|
||||
end
|
||||
|
||||
if isempty(results)
|
||||
println("ERROR: No matching test observations found")
|
||||
return nothing
|
||||
end
|
||||
|
||||
results_df = DataFrame(results)
|
||||
|
||||
# Compute and report metrics
|
||||
println()
|
||||
println("-"^70)
|
||||
println("HELD-OUT VALIDATION RESULTS")
|
||||
println("-"^70)
|
||||
|
||||
# Overall
|
||||
overall_r = cor(results_df.expert_val, results_df.model_val)
|
||||
overall_mae = mean(results_df.abs_error)
|
||||
overall_rmse = sqrt(mean(results_df.error .^ 2))
|
||||
println()
|
||||
println(@sprintf("Overall: r=%.4f, MAE=%.4f, RMSE=%.4f, n=%d",
|
||||
overall_r, overall_mae, overall_rmse, nrow(results_df)))
|
||||
|
||||
# By dimension
|
||||
println()
|
||||
println("By dimension:")
|
||||
for dim in sort(unique(results_df.dimension))
|
||||
d = filter(r -> r.dimension == dim, results_df)
|
||||
r_val = cor(d.expert_val, d.model_val)
|
||||
mae = mean(d.abs_error)
|
||||
rmse = sqrt(mean(d.error .^ 2))
|
||||
cov = count(skipmissing(d.covered_95)) / count(!ismissing, d.covered_95)
|
||||
println(@sprintf(" %-12s: r=%.4f, MAE=%.4f, RMSE=%.4f, CIC95=%.1f%%, n=%d",
|
||||
dim, r_val, mae, rmse, 100*cov, nrow(d)))
|
||||
end
|
||||
|
||||
# By source
|
||||
println()
|
||||
println("By source:")
|
||||
for var in sort(unique(results_df.var))
|
||||
v = filter(r -> r.var == var, results_df)
|
||||
nrow(v) < 5 && continue
|
||||
r_val = cor(v.expert_val, v.model_val)
|
||||
mae = mean(v.abs_error)
|
||||
println(@sprintf(" %-20s: r=%.4f, MAE=%.4f, n=%d", var, r_val, mae, nrow(v)))
|
||||
end
|
||||
|
||||
return results_df
|
||||
end
|
||||
|
||||
# =========================================================================
|
||||
# Main
|
||||
# =========================================================================
|
||||
|
||||
function main()
|
||||
args = ARGS
|
||||
|
||||
if isempty(args) || args[1] == "prepare"
|
||||
output_dir = prepare_holdout_data()
|
||||
|
||||
println()
|
||||
println("="^70)
|
||||
println("NEXT STEPS")
|
||||
println("="^70)
|
||||
println("""
|
||||
1. Run model on training data:
|
||||
julia 01_run_model.jl --data-dir $output_dir
|
||||
|
||||
2. Run post-estimation on training run output:
|
||||
julia 02_post_estimation.jl
|
||||
|
||||
3. Compute held-out metrics:
|
||||
julia scripts/validate_external.jl compute <party_positions_file.csv>
|
||||
""")
|
||||
|
||||
elseif args[1] == "compute" && length(args) >= 2
|
||||
model_file = args[2]
|
||||
test_dir = joinpath("validation", "external_split")
|
||||
|
||||
isfile(model_file) || error("Model file not found: $model_file")
|
||||
isdir(test_dir) || error("Test data not found. Run 'prepare' first.")
|
||||
|
||||
results_df = compute_holdout_metrics(model_file, test_dir)
|
||||
|
||||
if results_df !== nothing
|
||||
timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")
|
||||
output_file = joinpath("validation", "external_validation_$(timestamp).csv")
|
||||
CSV.write(output_file, results_df)
|
||||
println()
|
||||
println("Detailed results saved: $output_file")
|
||||
end
|
||||
|
||||
else
|
||||
println("Usage:")
|
||||
println(" julia scripts/validate_external.jl prepare")
|
||||
println(" julia scripts/validate_external.jl compute <party_positions.csv>")
|
||||
end
|
||||
end
|
||||
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
main()
|
||||
end
|
||||
@@ -0,0 +1,593 @@
|
||||
#!/usr/bin/env julia
|
||||
#############################################################################
|
||||
## validate_uncertainty.jl
|
||||
## Uncertainty validation: Posterior Predictive Coverage (PPC)
|
||||
##
|
||||
## Following Claassen (2019), this script computes:
|
||||
## - PPC: What % of expert values fall within 95% posterior predictive interval?
|
||||
## - Also computes 80% PPC for direct Claassen comparison (he reports 60.3%)
|
||||
## - Wilson score CI for coverage proportion
|
||||
## - Breakdown by survey source and decade
|
||||
##
|
||||
## Unlike credible interval coverage (which checks θ-CIs), posterior predictive
|
||||
## coverage simulates what a new expert observation would look like given the
|
||||
## model's beta likelihood, accounting for both position uncertainty AND
|
||||
## measurement noise. Well-calibrated models should yield ~95% PPC at 95%.
|
||||
##
|
||||
## No model re-run needed: reads θ, γ, and φ from existing chain CSV files.
|
||||
#############################################################################
|
||||
|
||||
using CSV, DataFrames, Statistics, Dates, Printf, Random, JSON
|
||||
|
||||
# =============================================================================
|
||||
# Utility: Wilson score CI for a proportion
|
||||
# =============================================================================
|
||||
|
||||
function wilson_ci(p, n; alpha=0.05)
|
||||
if n == 0
|
||||
return (lower=NaN, upper=NaN, se=NaN)
|
||||
end
|
||||
z = 1.96 # For 95% CI
|
||||
denominator = 1 + z^2/n
|
||||
center = (p + z^2/(2n)) / denominator
|
||||
margin = z * sqrt((p*(1-p) + z^2/(4n))/n) / denominator
|
||||
se = sqrt(p * (1-p) / n)
|
||||
return (lower=center - margin, upper=center + margin, se=se)
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 0: Find latest model run
|
||||
# =============================================================================
|
||||
|
||||
function find_latest_run(base_dir::String="model_outputs")
|
||||
if !isdir(base_dir)
|
||||
error("Model outputs directory not found: $base_dir")
|
||||
end
|
||||
runs = filter(d -> startswith(d, "run_") && isdir(joinpath(base_dir, d)), readdir(base_dir))
|
||||
if isempty(runs)
|
||||
error("No runs found in $base_dir")
|
||||
end
|
||||
sort!(runs, rev=true)
|
||||
latest = joinpath(base_dir, runs[1])
|
||||
println("Using latest run: $latest")
|
||||
return latest
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 1: Load expert_dim.csv from model run data
|
||||
# =============================================================================
|
||||
|
||||
function load_expert_dim(run_dir::String)
|
||||
expert_dim_file = joinpath(run_dir, "data", "expert_dim.csv")
|
||||
if !isfile(expert_dim_file)
|
||||
error("expert_dim.csv not found in $run_dir/data/")
|
||||
end
|
||||
expert_dim = CSV.read(expert_dim_file, DataFrame)
|
||||
println("Loaded expert_dim.csv: $(nrow(expert_dim)) observations")
|
||||
println(" Unique rr values: $(length(unique(expert_dim.rr_exp_dim)))")
|
||||
println(" Item indices (var_exp_dim): $(sort(unique(expert_dim.var_exp_dim)))")
|
||||
println(" Dimensions (dim_idx_exp): $(sort(unique(expert_dim.dim_idx_exp)))")
|
||||
return expert_dim
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 2: Selectively load chain columns
|
||||
# =============================================================================
|
||||
|
||||
function load_chains_selective(run_dir::String, needed_rr::Set{Int}, K::Int)
|
||||
"""Load only the chain columns we need for posterior predictive checks."""
|
||||
chains_dir = joinpath(run_dir, "chains")
|
||||
chain_files = sort(filter(f -> endswith(f, ".csv") && startswith(f, "chain_"), readdir(chains_dir)))
|
||||
|
||||
if isempty(chain_files)
|
||||
error("No chain files found in $chains_dir")
|
||||
end
|
||||
println("\nLoading $(length(chain_files)) chain files (selective columns)...")
|
||||
|
||||
# Build the set of column names we need
|
||||
needed_cols = Set{String}()
|
||||
|
||||
# theta columns: economic_lr.{rr} and galtan.{rr} for each unique rr
|
||||
for rr in needed_rr
|
||||
push!(needed_cols, "economic_lr.$rr")
|
||||
push!(needed_cols, "galtan.$rr")
|
||||
end
|
||||
|
||||
# Item parameters: gamma_exp_intercept.1-K, gamma_exp_slope.1-K
|
||||
for k in 1:K
|
||||
push!(needed_cols, "gamma_exp_intercept.$k")
|
||||
push!(needed_cols, "gamma_exp_slope.$k")
|
||||
end
|
||||
|
||||
# Precision parameter
|
||||
push!(needed_cols, "phi_exp_dim")
|
||||
|
||||
println(" Need $(length(needed_cols)) columns ($(length(needed_rr)) rr × 2 dims + $(2*K) item params + 1 phi)")
|
||||
|
||||
# Read the header from first chain to identify column indices
|
||||
first_chain_path = joinpath(chains_dir, chain_files[1])
|
||||
header_line = ""
|
||||
open(first_chain_path) do f
|
||||
for line in eachline(f)
|
||||
if !startswith(line, "#")
|
||||
header_line = line
|
||||
break
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
all_cols = split(header_line, ",")
|
||||
col_indices = Int[]
|
||||
col_names = String[]
|
||||
|
||||
for (i, col) in enumerate(all_cols)
|
||||
if col in needed_cols
|
||||
push!(col_indices, i)
|
||||
push!(col_names, col)
|
||||
end
|
||||
end
|
||||
|
||||
println(" Found $(length(col_indices))/$(length(needed_cols)) columns in chains")
|
||||
|
||||
if length(col_indices) < length(needed_cols)
|
||||
missing_cols = setdiff(needed_cols, Set(col_names))
|
||||
n_missing = length(missing_cols)
|
||||
sample = collect(missing_cols)[1:min(5, n_missing)]
|
||||
println(" WARNING: Missing columns (showing $( min(5, n_missing))/$n_missing): $sample")
|
||||
end
|
||||
|
||||
# Build a type specification for selective reading
|
||||
# We'll use CSV.read with select parameter
|
||||
select_symbols = Symbol.(col_names)
|
||||
|
||||
all_chains = DataFrame[]
|
||||
for (i, cf) in enumerate(chain_files)
|
||||
path = joinpath(chains_dir, cf)
|
||||
print(" Loading chain $i: $(cf)... ")
|
||||
t = @elapsed begin
|
||||
chain = CSV.read(path, DataFrame; comment="#", select=select_symbols)
|
||||
end
|
||||
println("$(nrow(chain)) samples, $(round(t, digits=1))s")
|
||||
push!(all_chains, chain)
|
||||
end
|
||||
|
||||
combined = vcat(all_chains...)
|
||||
println("Combined: $(nrow(combined)) total posterior draws")
|
||||
return combined
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 3: Compute posterior predictive coverage
|
||||
# =============================================================================
|
||||
|
||||
function compute_posterior_predictive_cic(chains::DataFrame, expert_dim::DataFrame;
|
||||
ci_level::Float64=0.95, seed::Int=42)
|
||||
"""
|
||||
Compute posterior predictive coverage for expert dimension observations.
|
||||
|
||||
For each expert observation n with observed value y_n:
|
||||
1. For each posterior draw s:
|
||||
- Get theta_s = theta[dim, rr] (on logit scale, but chains store inv_logit)
|
||||
- Compute mu_s = invlogit(gamma_intercept[k] + gamma_slope[k] * logit(theta_s))
|
||||
- V4 (Beta): Draw y_pred_s ~ Beta(phi * mu_s, phi * (1 - mu_s))
|
||||
- V5 (Beta-Binomial): Draw y_pred_s ~ Beta(phi * K * mu_s, phi * K * (1 - mu_s))
|
||||
where K = n_experts for that observation
|
||||
2. Compute quantile interval of y_pred draws
|
||||
3. Check if y_n falls within interval
|
||||
|
||||
Returns DataFrame with one row per observation plus coverage indicator.
|
||||
"""
|
||||
rng = MersenneTwister(seed)
|
||||
|
||||
alpha_lower = (1 - ci_level) / 2
|
||||
alpha_upper = 1 - alpha_lower
|
||||
|
||||
N = nrow(expert_dim)
|
||||
S = nrow(chains) # total posterior draws
|
||||
|
||||
# Detect V5 (Beta-Binomial with K-scaling) by presence of n_experts column
|
||||
has_k_scaling = hasproperty(expert_dim, :n_experts)
|
||||
if has_k_scaling
|
||||
k_vec = expert_dim.n_experts
|
||||
println("\nV5 detected: using Beta(phi*K*mu, phi*K*(1-mu)) with per-observation K")
|
||||
else
|
||||
println("\nV4 detected: using Beta(phi*mu, phi*(1-mu))")
|
||||
end
|
||||
|
||||
println("Computing posterior predictive coverage ($(round(Int, 100*ci_level))% level)")
|
||||
println(" Expert observations: $N")
|
||||
println(" Posterior draws: $S")
|
||||
|
||||
# Pre-extract phi vector
|
||||
phi_vec = chains[!, :phi_exp_dim]
|
||||
|
||||
# Pre-extract gamma vectors for each item k
|
||||
K = maximum(expert_dim.var_exp_dim)
|
||||
gamma_int = Dict{Int, Vector{Float64}}()
|
||||
gamma_slope = Dict{Int, Vector{Float64}}()
|
||||
for k in 1:K
|
||||
col_int = Symbol("gamma_exp_intercept.$k")
|
||||
col_slope = Symbol("gamma_exp_slope.$k")
|
||||
if hasproperty(chains, col_int) && hasproperty(chains, col_slope)
|
||||
gamma_int[k] = chains[!, col_int]
|
||||
gamma_slope[k] = chains[!, col_slope]
|
||||
end
|
||||
end
|
||||
|
||||
# Allocate result columns
|
||||
covered = BitVector(undef, N)
|
||||
pred_lower = Vector{Float64}(undef, N)
|
||||
pred_upper = Vector{Float64}(undef, N)
|
||||
pred_median = Vector{Float64}(undef, N)
|
||||
|
||||
# Pre-allocate per-observation draw buffer
|
||||
y_pred = Vector{Float64}(undef, S)
|
||||
|
||||
prog_interval = max(1, N ÷ 20)
|
||||
|
||||
for n in 1:N
|
||||
if n % prog_interval == 0 || n == N
|
||||
pct = round(100 * n / N, digits=1)
|
||||
print("\r Progress: $pct% ($n / $N)")
|
||||
end
|
||||
|
||||
rr = expert_dim.rr_exp_dim[n]
|
||||
dim = expert_dim.dim_idx_exp[n]
|
||||
k = expert_dim.var_exp_dim[n]
|
||||
y_obs = expert_dim.val[n]
|
||||
|
||||
# Get theta column (chains store inv_logit(theta), i.e. on [0,1] scale)
|
||||
theta_col = dim == 1 ? Symbol("economic_lr.$rr") : Symbol("galtan.$rr")
|
||||
|
||||
if !hasproperty(chains, theta_col) || !haskey(gamma_int, k)
|
||||
# Missing chain data — mark as not covered
|
||||
covered[n] = false
|
||||
pred_lower[n] = NaN
|
||||
pred_upper[n] = NaN
|
||||
pred_median[n] = NaN
|
||||
continue
|
||||
end
|
||||
|
||||
theta_star_vec = chains[!, theta_col] # inv_logit(theta), i.e. on [0,1]
|
||||
g_int = gamma_int[k]
|
||||
g_slope = gamma_slope[k]
|
||||
|
||||
# Effective concentration: phi for V4, phi * n_experts for V5
|
||||
k_mult = has_k_scaling ? Float64(k_vec[n]) : 1.0
|
||||
|
||||
# For each posterior draw, simulate a predictive observation
|
||||
for s in 1:S
|
||||
theta_star = theta_star_vec[s]
|
||||
|
||||
# Convert back to latent scale for linear predictor
|
||||
# theta_star is inv_logit(theta), so theta = logit(theta_star)
|
||||
# Clamp to avoid Inf
|
||||
theta_star_clamped = clamp(theta_star, 1e-10, 1 - 1e-10)
|
||||
theta_latent = log(theta_star_clamped / (1 - theta_star_clamped))
|
||||
|
||||
# Linear predictor
|
||||
lin = g_int[s] + g_slope[s] * theta_latent
|
||||
|
||||
# Mean of beta
|
||||
mu = 1 / (1 + exp(-lin))
|
||||
mu = clamp(mu, 1e-6, 1 - 1e-6)
|
||||
|
||||
# Beta parameters: phi * K * mu for V5, phi * mu for V4
|
||||
phi = phi_vec[s] * k_mult
|
||||
a = phi * mu
|
||||
b = phi * (1 - mu)
|
||||
|
||||
# Draw from Beta(a, b) via gamma method (no Distributions.jl needed)
|
||||
y_pred[s] = _rand_beta(rng, a, b)
|
||||
end
|
||||
|
||||
# Compute predictive interval
|
||||
sort!(y_pred)
|
||||
idx_lo = max(1, round(Int, alpha_lower * S))
|
||||
idx_hi = min(S, round(Int, alpha_upper * S))
|
||||
idx_med = round(Int, 0.5 * S)
|
||||
|
||||
pred_lower[n] = y_pred[idx_lo]
|
||||
pred_upper[n] = y_pred[idx_hi]
|
||||
pred_median[n] = y_pred[idx_med]
|
||||
covered[n] = (y_obs >= pred_lower[n]) && (y_obs <= pred_upper[n])
|
||||
end
|
||||
println() # newline after progress
|
||||
|
||||
# Add results to a copy of expert_dim
|
||||
result = DataFrame(
|
||||
rr = expert_dim.rr_exp_dim,
|
||||
dim_idx = expert_dim.dim_idx_exp,
|
||||
var_idx = expert_dim.var_exp_dim,
|
||||
val = expert_dim.val,
|
||||
party = expert_dim.party,
|
||||
country = expert_dim.country,
|
||||
year = expert_dim.year,
|
||||
project = expert_dim.project,
|
||||
var = expert_dim.var,
|
||||
pred_lower = pred_lower,
|
||||
pred_upper = pred_upper,
|
||||
pred_median = pred_median,
|
||||
covered = covered
|
||||
)
|
||||
|
||||
return result
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# Beta random variate without Distributions.jl
|
||||
# =============================================================================
|
||||
|
||||
"""
|
||||
_rand_beta(rng, a, b)
|
||||
|
||||
Generate a Beta(a, b) random variate using the Gamma method:
|
||||
Beta(a,b) = X/(X+Y) where X ~ Gamma(a), Y ~ Gamma(b).
|
||||
Uses Marsaglia & Tsang (2000) for Gamma generation.
|
||||
"""
|
||||
function _rand_beta(rng::AbstractRNG, a::Float64, b::Float64)
|
||||
x = _rand_gamma(rng, a)
|
||||
y = _rand_gamma(rng, b)
|
||||
return x / (x + y)
|
||||
end
|
||||
|
||||
"""
|
||||
_rand_gamma(rng, shape)
|
||||
|
||||
Generate Gamma(shape, 1) random variate using Marsaglia & Tsang (2000).
|
||||
For shape < 1, uses the rejection method with shape+1 then scales.
|
||||
"""
|
||||
function _rand_gamma(rng::AbstractRNG, shape::Float64)
|
||||
if shape < 1.0
|
||||
# Gamma(a) = Gamma(a+1) * U^(1/a) where U ~ Uniform(0,1)
|
||||
return _rand_gamma(rng, shape + 1.0) * rand(rng)^(1.0 / shape)
|
||||
end
|
||||
|
||||
# Marsaglia & Tsang (2000) for shape >= 1
|
||||
d = shape - 1.0/3.0
|
||||
c = 1.0 / sqrt(9.0 * d)
|
||||
|
||||
while true
|
||||
local x::Float64
|
||||
local v::Float64
|
||||
|
||||
while true
|
||||
x = randn(rng)
|
||||
v = 1.0 + c * x
|
||||
if v > 0.0
|
||||
break
|
||||
end
|
||||
end
|
||||
|
||||
v = v * v * v
|
||||
u = rand(rng)
|
||||
|
||||
if u < 1.0 - 0.0331 * x^2 * x^2
|
||||
return d * v
|
||||
end
|
||||
|
||||
if log(u) < 0.5 * x^2 + d * (1.0 - v + log(v))
|
||||
return d * v
|
||||
end
|
||||
end
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# STEP 4: Summarize and save results
|
||||
# =============================================================================
|
||||
|
||||
function summarize_coverage(result::DataFrame, ci_level::Float64)
|
||||
level_pct = round(Int, 100 * ci_level)
|
||||
println("\n" * "="^60)
|
||||
println("POSTERIOR PREDICTIVE COVERAGE ($level_pct%)")
|
||||
println("="^60)
|
||||
|
||||
# Overall by dimension
|
||||
dim_names = Dict(1 => "economic_lr", 2 => "galtan")
|
||||
|
||||
summary_rows = []
|
||||
for dim in sort(unique(result.dim_idx))
|
||||
subset = filter(r -> r.dim_idx == dim, result)
|
||||
n = nrow(subset)
|
||||
n_covered = sum(subset.covered)
|
||||
ppc = n_covered / n
|
||||
ci = wilson_ci(ppc, n)
|
||||
dim_name = dim_names[dim]
|
||||
|
||||
println(@sprintf("\n %-15s: %.1f%% [%.1f%%, %.1f%%] (%d/%d)",
|
||||
dim_name, 100*ppc, 100*ci.lower, 100*ci.upper, n_covered, n))
|
||||
|
||||
push!(summary_rows, (
|
||||
dimension = dim_name,
|
||||
cic = ppc,
|
||||
cic_pct = round(100 * ppc, digits=1),
|
||||
ci_lower = ci.lower,
|
||||
ci_upper = ci.upper,
|
||||
n = n,
|
||||
covered = n_covered
|
||||
))
|
||||
|
||||
# By project
|
||||
println("\n By survey source:")
|
||||
by_project = combine(groupby(subset, :project)) do df
|
||||
nc = sum(df.covered)
|
||||
DataFrame(n = nrow(df), covered = nc, cic = nc / nrow(df))
|
||||
end
|
||||
sort!(by_project, :n, rev=true)
|
||||
|
||||
@printf(" %-12s %6s %8s\n", "Project", "N", "PPC")
|
||||
for row in eachrow(by_project)
|
||||
@printf(" %-12s %6d %7.1f%%\n", row.project, row.n, 100*row.cic)
|
||||
end
|
||||
|
||||
# By decade
|
||||
subset_with_decade = copy(subset)
|
||||
subset_with_decade.decade = div.(subset_with_decade.year, 10) .* 10
|
||||
println("\n By decade:")
|
||||
by_decade = combine(groupby(subset_with_decade, :decade)) do df
|
||||
nc = sum(df.covered)
|
||||
DataFrame(n = nrow(df), covered = nc, cic = nc / nrow(df))
|
||||
end
|
||||
sort!(by_decade, :decade)
|
||||
|
||||
@printf(" %-8s %6s %8s\n", "Decade", "N", "PPC")
|
||||
for row in eachrow(by_decade)
|
||||
@printf(" %-8d %6d %7.1f%%\n", row.decade, row.n, 100*row.cic)
|
||||
end
|
||||
end
|
||||
|
||||
return summary_rows
|
||||
end
|
||||
|
||||
function save_results(result_95::DataFrame, summary_95, summary_80,
|
||||
by_project_95::Dict, output_dir::String="validation")
|
||||
if !isdir(output_dir)
|
||||
mkpath(output_dir)
|
||||
end
|
||||
|
||||
timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")
|
||||
|
||||
# Summary table (95%)
|
||||
if !isempty(summary_95)
|
||||
summary_df = DataFrame(summary_95)
|
||||
summary_file = joinpath(output_dir, "uncertainty_cic_summary_$timestamp.csv")
|
||||
CSV.write(summary_file, summary_df)
|
||||
println("\nSaved: $summary_file")
|
||||
end
|
||||
|
||||
# Also save 80% summary
|
||||
if !isempty(summary_80)
|
||||
summary80_df = DataFrame(summary_80)
|
||||
summary80_file = joinpath(output_dir, "uncertainty_cic_80pct_summary_$timestamp.csv")
|
||||
CSV.write(summary80_file, summary80_df)
|
||||
println("Saved: $summary80_file")
|
||||
end
|
||||
|
||||
# By-project tables (95%)
|
||||
dim_names = Dict(1 => "economic_lr", 2 => "galtan")
|
||||
for (dim, bp) in by_project_95
|
||||
project_file = joinpath(output_dir, "uncertainty_$(dim_names[dim])_by_project_$timestamp.csv")
|
||||
CSV.write(project_file, bp)
|
||||
println("Saved: $project_file")
|
||||
end
|
||||
|
||||
return summary_95
|
||||
end
|
||||
|
||||
function print_claassen_comparison(summary_95, summary_80)
|
||||
println("\n" * "="^60)
|
||||
println("COMPARISON WITH CLAASSEN (2019) BENCHMARKS")
|
||||
println("="^60)
|
||||
|
||||
println("\nClaassen's result:")
|
||||
println(" CIC (80% CI): 60.3%")
|
||||
println(" (Using credible intervals for θ, not posterior predictive)")
|
||||
println()
|
||||
println("Our results (posterior predictive):")
|
||||
println()
|
||||
|
||||
println("-"^60)
|
||||
@printf("%-15s %10s %10s %8s\n", "Dimension", "PPC 95%", "PPC 80%", "Status")
|
||||
println("-"^60)
|
||||
|
||||
dim_map_80 = Dict(r.dimension => r for r in summary_80)
|
||||
|
||||
for r in summary_95
|
||||
ppc80 = haskey(dim_map_80, r.dimension) ? dim_map_80[r.dimension].cic : NaN
|
||||
# Well-calibrated: 95% PPC should be near 95%
|
||||
status = r.cic >= 0.90 ? "GOOD" : (r.cic >= 0.80 ? "OK" : "LOW")
|
||||
@printf("%-15s %9.1f%% %9.1f%% %8s\n",
|
||||
r.dimension, 100*r.cic, 100*ppc80, status)
|
||||
end
|
||||
println("-"^60)
|
||||
println()
|
||||
println("Interpretation:")
|
||||
println(" 95% PPC ~95% = well-calibrated uncertainty")
|
||||
println(" 80% PPC > 60% = exceeds Claassen (2019) benchmark")
|
||||
end
|
||||
|
||||
# =============================================================================
|
||||
# MAIN
|
||||
# =============================================================================
|
||||
|
||||
function main()
|
||||
println("="^60)
|
||||
println("UNCERTAINTY VALIDATION: Posterior Predictive Coverage")
|
||||
println("="^60)
|
||||
println("Following Claassen (2019) validation framework")
|
||||
println("Posterior predictive intervals account for both position")
|
||||
println("uncertainty AND observation-level measurement noise.")
|
||||
println()
|
||||
|
||||
# Step 0: Parse options and find run directory
|
||||
run_dir = nothing
|
||||
quick_mode = get(ENV, "QUICK_VALIDATION", "0") == "1"
|
||||
for (i, arg) in enumerate(ARGS)
|
||||
if arg == "--run-dir" && i < length(ARGS)
|
||||
run_dir = ARGS[i + 1]
|
||||
elseif startswith(arg, "--run-dir=")
|
||||
run_dir = split(arg, "=", limit=2)[2]
|
||||
elseif arg == "--quick"
|
||||
quick_mode = true
|
||||
end
|
||||
end
|
||||
if run_dir === nothing
|
||||
run_dir = find_latest_run()
|
||||
else
|
||||
println("Using specified run directory: $run_dir")
|
||||
end
|
||||
|
||||
# Step 1: Load expert_dim.csv
|
||||
expert_dim = load_expert_dim(run_dir)
|
||||
|
||||
# Step 2: Selectively load chains
|
||||
needed_rr = Set(expert_dim.rr_exp_dim)
|
||||
K = maximum(expert_dim.var_exp_dim)
|
||||
chains = load_chains_selective(run_dir, needed_rr, K)
|
||||
|
||||
# Step 3a: Compute 95% posterior predictive coverage
|
||||
result_95 = compute_posterior_predictive_cic(chains, expert_dim; ci_level=0.95)
|
||||
summary_95 = summarize_coverage(result_95, 0.95)
|
||||
|
||||
# Step 3b: Compute 80% posterior predictive coverage (Claassen benchmark)
|
||||
# Recompute coverage from the same predictive draws but with 80% quantiles
|
||||
if quick_mode
|
||||
println("\nQUICK MODE: skipping 80% PPC recomputation")
|
||||
summary_80 = [(dimension=r.dimension, n=r.n, covered=r.covered, cic=NaN, ci_level=0.80) for r in summary_95]
|
||||
else
|
||||
println("\n" * "="^60)
|
||||
println("RECOMPUTING WITH 80% LEVEL (Claassen comparison)")
|
||||
println("="^60)
|
||||
result_80 = compute_posterior_predictive_cic(chains, expert_dim; ci_level=0.80, seed=42)
|
||||
summary_80 = summarize_coverage(result_80, 0.80)
|
||||
end
|
||||
|
||||
# Build by-project tables for 95%
|
||||
dim_names = Dict(1 => "economic_lr", 2 => "galtan")
|
||||
by_project_95 = Dict{Int, DataFrame}()
|
||||
for dim in sort(unique(result_95.dim_idx))
|
||||
subset = filter(r -> r.dim_idx == dim, result_95)
|
||||
bp = combine(groupby(subset, :project)) do df
|
||||
nc = sum(df.covered)
|
||||
DataFrame(n = nrow(df), covered = nc, cic = nc / nrow(df))
|
||||
end
|
||||
sort!(bp, :n, rev=true)
|
||||
by_project_95[dim] = bp
|
||||
end
|
||||
|
||||
# Step 4: Save results
|
||||
save_results(result_95, summary_95, summary_80, by_project_95)
|
||||
|
||||
# Step 5: Print Claassen comparison
|
||||
print_claassen_comparison(summary_95, summary_80)
|
||||
|
||||
println("\n" * "="^60)
|
||||
println("VALIDATION COMPLETE")
|
||||
println("="^60)
|
||||
|
||||
return (summary_95=summary_95, summary_80=summary_80)
|
||||
end
|
||||
|
||||
if abspath(PROGRAM_FILE) == @__FILE__
|
||||
main()
|
||||
end
|
||||
Reference in New Issue
Block a user