#!/usr/bin/env julia
#############################################################################
## validate_construct.jl
## Construct validity: Party family ordering and temporal stability
##
## Following Claassen (2019), this script validates:
## 1. Party family ordering: Do family means follow theoretically expected orderings?
## 2. Temporal stability: Flag parties with implausible position changes
##
## Uses ParlGov party family classifications (Döring & Manow 2024) via PartyFacts IDs.
#############################################################################

using CSV, DataFrames, Statistics, StatsBase, Dates, Printf

# Family code → display name mapping
const FAMILY_DISPLAY_NAMES = Dict(
    "com" => "Communist/Far Left",
    "eco" => "Green/Ecological",
    "soc" => "Social Democratic",
    "lib" => "Liberal",
    "chr" => "Christian Democratic",
    "con" => "Conservative",
    "right" => "Radical Right"
)

# Substantive families (drop Specialist, Other, Agrarian — heterogeneous or ambiguous)
const SUBSTANTIVE_FAMILIES = Set(["com", "eco", "soc", "lib", "chr", "con", "right"])

# Expected orderings (theoretically motivated)
# Economic: Communist < Social Democratic < Green < Christian Democratic < Conservative
# (5-family core — Liberal position is ambiguous cross-nationally)
const EXPECTED_ECONOMIC_ORDER = ["com", "soc", "eco", "chr", "con"]

# Cultural: Green < Liberal < Social Democratic < Christian Democratic < Conservative < Radical Right
const EXPECTED_GALTAN_ORDER = ["eco", "lib", "soc", "chr", "con", "right"]

function load_model_output(base_dir::String=".")
    """Load the most recent 2D model party positions output"""

    position_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
                           !endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(base_dir))
    legacy_files = filter(f -> startswith(f, "party_positions_v1_") && endswith(f, ".csv"), readdir(base_dir))
    append!(position_files, legacy_files)

    if !isempty(position_files)
        latest = sort(position_files)[end]
        println("Loading model output: $latest")
        return CSV.read(joinpath(base_dir, latest), DataFrame), latest
    end

    # Check output estimations directory
    est_dir = joinpath(base_dir, "outputs", "estimations", "latest")
    if isdir(est_dir)
        est_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
                           !endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(est_dir))
        if !isempty(est_files)
            latest = sort(est_files)[end]
            println("Loading model output: outputs/estimations/latest/$latest")
            return CSV.read(joinpath(est_dir, latest), DataFrame), latest
        end
    end

    error("No party_positions_*.csv found. Run 02_post_estimation.jl first.")
end

function validate_party_families(model::DataFrame)
    """Check whether party family means follow theoretically expected orderings"""

    println("\n" * "="^60)
    println("PARTY FAMILY ORDERING VALIDATION")
    println("="^60)
    println("\nUsing ParlGov family classifications (Döring & Manow 2024)")
    println()

    party_col = hasproperty(model, :party_id) ? :party_id : :party

    # Load party families
    families_df = CSV.read("data/party_families.csv", DataFrame)

    # Join to model output
    model_with_families = innerjoin(model, families_df, on=party_col => :partyfacts_id)

    # Filter to substantive families
    filter!(r -> r.family in SUBSTANTIVE_FAMILIES, model_with_families)

    n_parties = length(unique(model_with_families[!, party_col]))
    n_obs = nrow(model_with_families)
    println("  Matched $n_parties parties ($n_obs party-years) across $(length(SUBSTANTIVE_FAMILIES)) families")
    println()

    # Compute family means
    family_stats = combine(groupby(model_with_families, :family)) do df
        DataFrame(
            n_parties = length(unique(df[!, party_col])),
            n_obs = nrow(df),
            mean_economic = mean(df.economic_lr),
            sd_economic = std(df.economic_lr),
            mean_galtan = mean(df.galtan),
            sd_galtan = std(df.galtan)
        )
    end

    # Add display names
    family_stats.family_name = [get(FAMILY_DISPLAY_NAMES, f, f) for f in family_stats.family]

    # Sort by economic mean for display
    sort!(family_stats, :mean_economic)

    # Print table
    @printf("  %-22s %7s %7s %10s %10s %10s %10s\n",
        "Family", "Parties", "Obs", "Econ Mean", "Econ SD", "Cult Mean", "Cult SD")
    println("  " * "-"^76)

    for row in eachrow(family_stats)
        @printf("  %-22s %7d %7d %10.3f %10.3f %10.3f %10.3f\n",
            row.family_name, row.n_parties, row.n_obs,
            row.mean_economic, row.sd_economic, row.mean_galtan, row.sd_galtan)
    end

    # Compute Spearman rank correlations for expected orderings
    println()

    # Economic ordering
    econ_lookup = Dict(row.family => row.mean_economic for row in eachrow(family_stats))
    econ_observed = [econ_lookup[f] for f in EXPECTED_ECONOMIC_ORDER if haskey(econ_lookup, f)]
    econ_expected_ranks = collect(1:length(econ_observed))
    econ_observed_ranks = ordinalrank(econ_observed)
    rho_econ = corspearman(Float64.(econ_expected_ranks), Float64.(econ_observed_ranks))

    println(@sprintf("  Economic ordering (5-family core): Spearman ρ = %.3f", rho_econ))
    econ_families_used = [f for f in EXPECTED_ECONOMIC_ORDER if haskey(econ_lookup, f)]
    econ_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in econ_families_used]
    println("    Expected: ", join(econ_names, " < "))
    observed_econ_order = econ_families_used[sortperm(econ_observed)]
    observed_econ_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in observed_econ_order]
    println("    Observed: ", join(observed_econ_names, " < "))

    # Cultural ordering
    galtan_lookup = Dict(row.family => row.mean_galtan for row in eachrow(family_stats))
    galtan_observed = [galtan_lookup[f] for f in EXPECTED_GALTAN_ORDER if haskey(galtan_lookup, f)]
    galtan_expected_ranks = collect(1:length(galtan_observed))
    galtan_observed_ranks = ordinalrank(galtan_observed)
    rho_galtan = corspearman(Float64.(galtan_expected_ranks), Float64.(galtan_observed_ranks))

    println(@sprintf("  Cultural ordering (6-family): Spearman ρ = %.3f", rho_galtan))
    galtan_families_used = [f for f in EXPECTED_GALTAN_ORDER if haskey(galtan_lookup, f)]
    galtan_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in galtan_families_used]
    println("    Expected: ", join(galtan_names, " < "))
    observed_galtan_order = galtan_families_used[sortperm(galtan_observed)]
    observed_galtan_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in observed_galtan_order]
    println("    Observed: ", join(observed_galtan_names, " < "))

    println()
    println("-"^60)
    if rho_econ >= 0.9 && rho_galtan >= 0.8
        println("EXCELLENT: Family means follow expected orderings on both dimensions")
    elseif rho_econ >= 0.7 && rho_galtan >= 0.7
        println("GOOD: Family means broadly follow expected orderings")
    else
        println("CONCERN: Review family ordering results")
    end

    return family_stats, rho_econ, rho_galtan
end

function validate_temporal_stability(model::DataFrame)
    """Check for implausible year-to-year position changes"""

    println("\n" * "="^60)
    println("TEMPORAL STABILITY VALIDATION")
    println("="^60)
    println("\nFlagging parties with >0.10 change per year")
    println()

    party_col = hasproperty(model, :party_id) ? :party_id : :party

    # Compute year-to-year changes within each party
    sort!(model, [party_col, :year])

    unstable_parties = []

    for party_df in groupby(model, party_col)
        if nrow(party_df) < 2
            continue
        end

        party_id = party_df[1, party_col]
        country = party_df[1, :country]

        # Compute differences
        for dim in [:economic_lr, :galtan]
            vals = party_df[!, dim]
            years = party_df.year

            for i in 2:length(vals)
                diff = abs(vals[i] - vals[i-1])
                year_gap = years[i] - years[i-1]

                # Normalize by year gap (handle multi-year gaps)
                annual_change = diff / max(year_gap, 1)

                if annual_change > 0.10
                    push!(unstable_parties, (
                        party_id = party_id,
                        country = country,
                        dimension = string(dim),
                        year_from = years[i-1],
                        year_to = years[i],
                        val_from = vals[i-1],
                        val_to = vals[i],
                        change = diff,
                        annual_change = annual_change
                    ))
                end
            end
        end
    end

    if isempty(unstable_parties)
        println("  No parties with >0.10 annual change found")
        println("  EXCELLENT: Positions are temporally stable")
        return DataFrame()
    end

    unstable_df = DataFrame(unstable_parties)
    sort!(unstable_df, :annual_change, rev=true)

    println("  Found $(nrow(unstable_df)) instances of rapid change:")
    println()
    @printf("  %-8s %-8s %-12s %-10s %-10s %8s\n",
        "Party", "Country", "Dimension", "Years", "Change", "Annual")
    println("  " * "-"^60)

    for row in eachrow(unstable_df[1:min(20, nrow(unstable_df)), :])
        @printf("  %-8d %-8s %-12s %d->%d %8.3f %8.3f\n",
            row.party_id, row.country, row.dimension,
            row.year_from, row.year_to, row.change, row.annual_change)
    end

    if nrow(unstable_df) > 20
        println("  ... and $(nrow(unstable_df) - 20) more")
    end

    println()
    println("-"^60)
    n_parties = length(unique(unstable_df.party_id))
    n_total = length(unique(model[!, party_col]))
    println(@sprintf("Unstable parties: %d/%d (%.1f%%)", n_parties, n_total, 100*n_parties/n_total))

    return unstable_df
end

function validate_position_distributions(model::DataFrame)
    """Check overall distribution of positions makes sense"""

    println("\n" * "="^60)
    println("POSITION DISTRIBUTION VALIDATION")
    println("="^60)
    println("\nSummary statistics for model estimates")
    println()

    for dim in [:economic_lr, :galtan]
        if !hasproperty(model, dim)
            continue
        end

        vals = model[!, dim]
        println("$dim:")
        println(@sprintf("  Mean:   %.3f (should be ~0.50)", mean(vals)))
        println(@sprintf("  Median: %.3f (should be ~0.50)", median(vals)))
        println(@sprintf("  SD:     %.3f (should be ~0.15)", std(vals)))
        println(@sprintf("  Min:    %.3f", minimum(vals)))
        println(@sprintf("  Max:    %.3f", maximum(vals)))
        println(@sprintf("  Q25:    %.3f", quantile(vals, 0.25)))
        println(@sprintf("  Q75:    %.3f", quantile(vals, 0.75)))
        println()
    end

    # Check for extreme values
    println("Extreme positions (< 0.10 or > 0.90):")

    party_col = hasproperty(model, :party_id) ? :party_id : :party

    for dim in [:economic_lr, :galtan]
        if !hasproperty(model, dim)
            continue
        end

        extreme = filter(row -> row[dim] < 0.10 || row[dim] > 0.90, model)
        n_extreme = nrow(extreme)
        pct_extreme = 100 * n_extreme / nrow(model)

        println(@sprintf("  %s: %d (%.1f%%)", dim, n_extreme, pct_extreme))

        if n_extreme > 0 && n_extreme <= 10
            for row in eachrow(extreme[1:min(5, nrow(extreme)), :])
                println(@sprintf("    Party %d (%s) %d: %.3f",
                    row[party_col], row.country, row.year, row[dim]))
            end
        end
    end
end

function validate_country_patterns(model::DataFrame)
    """Check country-level patterns make sense"""

    println("\n" * "="^60)
    println("COUNTRY-LEVEL VALIDATION")
    println("="^60)
    println("\nMean positions by country (should vary but not wildly)")
    println()

    country_stats = combine(groupby(model, :country)) do df
        DataFrame(
            n_parties = length(unique(hasproperty(df, :party_id) ? df.party_id : df.party)),
            n_obs = nrow(df),
            mean_econ = mean(df.economic_lr),
            mean_galtan = mean(df.galtan),
            sd_econ = std(df.economic_lr),
            sd_galtan = std(df.galtan)
        )
    end

    sort!(country_stats, :n_obs, rev=true)

    @printf("%-4s %6s %6s %8s %8s %8s %8s\n",
        "CC", "Parties", "N", "Econ", "SD", "Galtan", "SD")
    println("-"^60)

    for row in eachrow(country_stats[1:min(20, nrow(country_stats)), :])
        @printf("%-4s %6d %6d %8.3f %8.3f %8.3f %8.3f\n",
            row.country, row.n_parties, row.n_obs,
            row.mean_econ, row.sd_econ, row.mean_galtan, row.sd_galtan)
    end

    # Flag countries with unusual patterns
    println("\nCountries with unusual patterns:")
    unusual = filter(row -> row.mean_econ < 0.35 || row.mean_econ > 0.65 ||
                           row.mean_galtan < 0.35 || row.mean_galtan > 0.65, country_stats)

    if nrow(unusual) == 0
        println("  None - all countries have balanced party systems")
    else
        for row in eachrow(unusual)
            issues = String[]
            if row.mean_econ < 0.35
                push!(issues, "left-skewed economy")
            elseif row.mean_econ > 0.65
                push!(issues, "right-skewed economy")
            end
            if row.mean_galtan < 0.35
                push!(issues, "cultural-skewed")
            elseif row.mean_galtan > 0.65
                push!(issues, "TAN-skewed")
            end
            println("  $(row.country): $(join(issues, ", "))")
        end
    end

    return country_stats
end

function save_construct_results(families::DataFrame, unstable::DataFrame,
                                countries::DataFrame, output_dir::String="outputs/checks")
    """Save construct validation results"""

    if !isdir(output_dir)
        mkpath(output_dir)
    end

    timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")

    if nrow(families) > 0
        families_file = joinpath(output_dir, "construct_families_$timestamp.csv")
        CSV.write(families_file, families)
        println("\nSaved: $families_file")
    end

    if nrow(unstable) > 0
        unstable_file = joinpath(output_dir, "construct_unstable_$timestamp.csv")
        CSV.write(unstable_file, unstable)
        println("Saved: $unstable_file")
    end

    if nrow(countries) > 0
        country_file = joinpath(output_dir, "construct_countries_$timestamp.csv")
        CSV.write(country_file, countries)
        println("Saved: $country_file")
    end
end


# Main execution
function main()
    println("="^60)
    println("CONSTRUCT VALIDITY: Face Validity Checks")
    println("="^60)
    println("Checking if model estimates match expectations")
    println()

    # Load data
    model, model_file = load_model_output()

    println("\nModel output:")
    println("  Rows: $(nrow(model))")
    println("  Columns: $(names(model))")

    # Run validations
    families, rho_econ, rho_galtan = validate_party_families(model)
    unstable = validate_temporal_stability(model)
    validate_position_distributions(model)
    countries = validate_country_patterns(model)

    # Save results
    save_construct_results(families, unstable, countries)

    # Summary
    println("\n" * "="^60)
    println("CONSTRUCT VALIDITY SUMMARY")
    println("="^60)

    println(@sprintf("  Party family ordering:"))
    println(@sprintf("    Economic (5-family): Spearman ρ = %.3f", rho_econ))
    println(@sprintf("    Cultural (6-family):  Spearman ρ = %.3f", rho_galtan))

    n_unstable = nrow(unstable) > 0 ? length(unique(unstable.party_id)) : 0
    party_col = hasproperty(model, :party_id) ? :party_id : :party
    n_total = length(unique(model[!, party_col]))
    println(@sprintf("  Temporal stability: %d/%d parties stable (>0.10/yr threshold)",
        n_total - n_unstable, n_total))

    if rho_econ >= 0.9 && rho_galtan >= 0.8 && n_unstable < 0.1 * n_total
        println("\n  EXCELLENT: Model has good construct validity")
    elseif rho_econ >= 0.7 && rho_galtan >= 0.7
        println("\n  GOOD: Model has reasonable construct validity")
    else
        println("\n  CONCERN: Review family ordering results")
    end


    println("\n" * "="^60)
    println("VALIDATION COMPLETE")
    println("="^60)

    return (families=families, unstable=unstable, countries=countries)
end

if abspath(PROGRAM_FILE) == @__FILE__
    main()
end
