#!/usr/bin/env julia
#############################################################################
## validate_convergent.jl
## Convergent validity: Compare model estimates to external expert surveys
##
## Following Claassen (2019), this script computes:
## - Pearson/Spearman correlations between model and expert estimates
## - Fisher z-transformation for correlation confidence intervals
## - Mean Absolute Error (MAE) and Root Mean Square Error (RMSE)
## - Breakdown by survey project and decade
##
## Target: r > 0.8 with CHES (Claassen achieved 0.50-0.57)
#############################################################################

using CSV, DataFrames, Statistics, StatsBase, Dates, Printf, JSON

# Fisher z-transformation for correlation confidence intervals
fisher_z(r) = 0.5 * log((1 + r) / (1 - r))
fisher_z_inv(z) = (exp(2z) - 1) / (exp(2z) + 1)

function correlation_ci(r, n; alpha=0.05)
    """Calculate confidence interval for correlation using Fisher z-transformation"""
    if n < 4
        return (lower=NaN, upper=NaN)
    end
    z = fisher_z(r)
    se = 1 / sqrt(n - 3)
    z_crit = 1.96  # For 95% CI
    z_lower = z - z_crit * se
    z_upper = z + z_crit * se
    return (lower=fisher_z_inv(z_lower), upper=fisher_z_inv(z_upper))
end

function load_model_output(base_dir::String=".")
    """Load the most recent 2D model party positions output"""

    # First check for post_estimation output in root (current name, with legacy fallback)
    position_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
                           !endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(base_dir))
    legacy_files = filter(f -> startswith(f, "party_positions_v1_") && endswith(f, ".csv"), readdir(base_dir))
    append!(position_files, legacy_files)

    if !isempty(position_files)
        latest = sort(position_files)[end]
        println("Loading model output: $latest")
        return CSV.read(joinpath(base_dir, latest), DataFrame), latest
    end

    # Check current pipeline output directory, with legacy estimations/ fallback
    for (label, est_dir) in [
        ("outputs/estimations/latest", joinpath(base_dir, "outputs", "estimations", "latest")),
        ("estimations", joinpath(base_dir, "estimations")),
    ]
        if isdir(est_dir)
            est_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
                               !endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(est_dir))
            if !isempty(est_files)
                latest = sort(est_files)[end]
                println("Loading model output: $label/$latest")
                return CSV.read(joinpath(est_dir, latest), DataFrame), latest
            end
        end
    end

    error("No party_positions_*.csv found. Run 02_post_estimation.jl first.")
end

function load_expert_data(base_dir::String=".")
    """Load expert survey data files"""

    data_dir = isfile(joinpath(base_dir, "expert.csv")) ? base_dir : joinpath(base_dir, "data")

    # Load dimension-specific expert data
    expert_file = joinpath(data_dir, "expert.csv")
    if !isfile(expert_file)
        error("expert.csv not found in $base_dir or $(joinpath(base_dir, "data"))")
    end

    println("Loading expert.csv...")
    expert = CSV.read(expert_file, DataFrame)
    println("  Rows: $(nrow(expert))")
    println("  Variables: $(unique(expert.var))")

    # Load L-R data
    lr_file = joinpath(data_dir, "lr_data.csv")
    if !isfile(lr_file)
        error("lr_data.csv not found in $base_dir or $(joinpath(base_dir, "data"))")
    end

    println("Loading lr_data.csv...")
    lr_data = CSV.read(lr_file, DataFrame)
    println("  Rows: $(nrow(lr_data))")
    println("  Variables: $(unique(lr_data.var))")

    return expert, lr_data
end

function validate_economic_lr(model::DataFrame, expert::DataFrame)
    """Validate economic_lr against CHES/V-Party/POPPA/GPS lrecon"""

    println("\n" * "="^60)
    println("CONVERGENT VALIDITY: economic_lr")
    println("="^60)

    # Filter expert data for economic dimension
    econ_vars = filter(v -> startswith(v, "lrecon_"), unique(expert.var))
    econ_expert = filter(row -> row.var in econ_vars, expert)

    println("\nExpert data variables: $(econ_vars)")
    println("Expert observations: $(nrow(econ_expert))")

    # Merge with model output
    # Model has party_id column (from segment-based), expert has party column
    if hasproperty(model, :party_id)
        model_merge = select(model, :party_id => :party, :year, :economic_lr, :economic_lr_se)
    else
        model_merge = select(model, :party, :year, :economic_lr, :economic_lr_se)
    end

    merged = innerjoin(econ_expert, model_merge, on=[:party, :year])

    println("Merged observations: $(nrow(merged))")

    if nrow(merged) < 10
        println("WARNING: Too few observations for meaningful validation")
        return nothing
    end

    # Compute overall correlation
    r_pearson = cor(merged.val, merged.economic_lr)
    r_spearman = corspearman(merged.val, merged.economic_lr)
    mae = mean(abs.(merged.val .- merged.economic_lr))
    rmse = sqrt(mean((merged.val .- merged.economic_lr).^2))

    ci = correlation_ci(r_pearson, nrow(merged))

    println("\n--- Overall Statistics ---")
    println(@sprintf("  Pearson r:  %.4f [%.4f, %.4f]", r_pearson, ci.lower, ci.upper))
    println(@sprintf("  Spearman r: %.4f", r_spearman))
    println(@sprintf("  MAE:        %.4f", mae))
    println(@sprintf("  RMSE:       %.4f", rmse))
    println(@sprintf("  N:          %d", nrow(merged)))

    # Breakdown by project
    println("\n--- By Project ---")
    by_project = combine(groupby(merged, :project)) do df
        n = nrow(df)
        if n < 3
            return DataFrame(n=n, r_pearson=NaN, mae=NaN)
        end
        DataFrame(
            n = n,
            r_pearson = cor(df.val, df.economic_lr),
            r_spearman = corspearman(df.val, df.economic_lr),
            mae = mean(abs.(df.val .- df.economic_lr)),
            rmse = sqrt(mean((df.val .- df.economic_lr).^2))
        )
    end

    for row in eachrow(sort(by_project, :n, rev=true))
        if !isnan(row.r_pearson)
            println(@sprintf("  %-10s: r=%.3f, MAE=%.3f, n=%d",
                row.project, row.r_pearson, row.mae, row.n))
        end
    end

    # Breakdown by decade
    println("\n--- By Decade ---")
    merged.decade = div.(merged.year, 10) .* 10
    by_decade = combine(groupby(merged, :decade)) do df
        n = nrow(df)
        if n < 3
            return DataFrame(n=n, r_pearson=NaN, mae=NaN)
        end
        DataFrame(
            n = n,
            r_pearson = cor(df.val, df.economic_lr),
            mae = mean(abs.(df.val .- df.economic_lr))
        )
    end

    for row in eachrow(sort(by_decade, :decade))
        if !isnan(row.r_pearson)
            println(@sprintf("  %ds: r=%.3f, MAE=%.3f, n=%d",
                row.decade, row.r_pearson, row.mae, row.n))
        end
    end

    return (
        dimension = "economic_lr",
        r_pearson = r_pearson,
        r_spearman = r_spearman,
        ci_lower = ci.lower,
        ci_upper = ci.upper,
        mae = mae,
        rmse = rmse,
        n = nrow(merged),
        by_project = by_project,
        by_decade = by_decade
    )
end

function validate_galtan(model::DataFrame, expert::DataFrame)
    """Validate galtan against CHES galtan and V-Party/GPS libcons"""

    println("\n" * "="^60)
    println("CONVERGENT VALIDITY: galtan")
    println("="^60)

    # Filter expert data for GAL-TAN dimension
    galtan_vars = filter(v -> occursin("galtan", v) || occursin("libcon", v), unique(expert.var))
    galtan_expert = filter(row -> row.var in galtan_vars, expert)

    println("\nExpert data variables: $(galtan_vars)")
    println("Expert observations: $(nrow(galtan_expert))")

    # Merge with model output
    if hasproperty(model, :party_id)
        model_merge = select(model, :party_id => :party, :year, :galtan, :galtan_se)
    else
        model_merge = select(model, :party, :year, :galtan, :galtan_se)
    end

    merged = innerjoin(galtan_expert, model_merge, on=[:party, :year])

    println("Merged observations: $(nrow(merged))")

    if nrow(merged) < 10
        println("WARNING: Too few observations for meaningful validation")
        return nothing
    end

    # Compute overall correlation
    r_pearson = cor(merged.val, merged.galtan)
    r_spearman = corspearman(merged.val, merged.galtan)
    mae = mean(abs.(merged.val .- merged.galtan))
    rmse = sqrt(mean((merged.val .- merged.galtan).^2))

    ci = correlation_ci(r_pearson, nrow(merged))

    println("\n--- Overall Statistics ---")
    println(@sprintf("  Pearson r:  %.4f [%.4f, %.4f]", r_pearson, ci.lower, ci.upper))
    println(@sprintf("  Spearman r: %.4f", r_spearman))
    println(@sprintf("  MAE:        %.4f", mae))
    println(@sprintf("  RMSE:       %.4f", rmse))
    println(@sprintf("  N:          %d", nrow(merged)))

    # Breakdown by project
    println("\n--- By Project ---")
    by_project = combine(groupby(merged, :project)) do df
        n = nrow(df)
        if n < 3
            return DataFrame(n=n, r_pearson=NaN, mae=NaN)
        end
        DataFrame(
            n = n,
            r_pearson = cor(df.val, df.galtan),
            r_spearman = corspearman(df.val, df.galtan),
            mae = mean(abs.(df.val .- df.galtan)),
            rmse = sqrt(mean((df.val .- df.galtan).^2))
        )
    end

    for row in eachrow(sort(by_project, :n, rev=true))
        if !isnan(row.r_pearson)
            println(@sprintf("  %-10s: r=%.3f, MAE=%.3f, n=%d",
                row.project, row.r_pearson, row.mae, row.n))
        end
    end

    # Breakdown by decade
    println("\n--- By Decade ---")
    merged.decade = div.(merged.year, 10) .* 10
    by_decade = combine(groupby(merged, :decade)) do df
        n = nrow(df)
        if n < 3
            return DataFrame(n=n, r_pearson=NaN, mae=NaN)
        end
        DataFrame(
            n = n,
            r_pearson = cor(df.val, df.galtan),
            mae = mean(abs.(df.val .- df.galtan))
        )
    end

    for row in eachrow(sort(by_decade, :decade))
        if !isnan(row.r_pearson)
            println(@sprintf("  %ds: r=%.3f, MAE=%.3f, n=%d",
                row.decade, row.r_pearson, row.mae, row.n))
        end
    end

    return (
        dimension = "galtan",
        r_pearson = r_pearson,
        r_spearman = r_spearman,
        ci_lower = ci.lower,
        ci_upper = ci.upper,
        mae = mae,
        rmse = rmse,
        n = nrow(merged),
        by_project = by_project,
        by_decade = by_decade
    )
end

function validate_discriminant(model::DataFrame, expert::DataFrame)
    """Compute cross-dimension correlations for discriminant validity (Campbell & Fiske 1959 MTMM)"""

    println("\n" * "="^60)
    println("DISCRIMINANT VALIDITY: Cross-dimension correlations")
    println("="^60)
    println("\nCampbell & Fiske (1959) MTMM framework:")
    println("  Convergent: same dimension, different method → HIGH")
    println("  Discriminant: different dimension, different method → LOW")
    println()

    # Get party column
    if hasproperty(model, :party_id)
        model_econ = select(model, :party_id => :party, :year, :economic_lr)
        model_gal = select(model, :party_id => :party, :year, :galtan)
    else
        model_econ = select(model, :party, :year, :economic_lr)
        model_gal = select(model, :party, :year, :galtan)
    end

    results = []

    # 1. Expert economic vs Model economic (convergent - already computed, include for matrix)
    econ_vars = filter(v -> startswith(v, "lrecon_"), unique(expert.var))
    econ_expert = filter(row -> row.var in econ_vars, expert)
    merged_ee = innerjoin(econ_expert, model_econ, on=[:party, :year])
    if nrow(merged_ee) >= 10
        r = cor(merged_ee.val, merged_ee.economic_lr)
        push!(results, (model_dim="economic_lr", expert_dim="economic",
                        r_pearson=r, r_spearman=corspearman(merged_ee.val, merged_ee.economic_lr),
                        n=nrow(merged_ee), type="convergent"))
        @printf("  Model Economic  × Expert Economic:  r = %.3f (convergent, n=%d)\n", r, nrow(merged_ee))
    end

    # 2. Expert economic vs Model galtan (discriminant)
    merged_eg = innerjoin(econ_expert, model_gal, on=[:party, :year])
    if nrow(merged_eg) >= 10
        r = cor(merged_eg.val, merged_eg.galtan)
        push!(results, (model_dim="galtan", expert_dim="economic",
                        r_pearson=r, r_spearman=corspearman(merged_eg.val, merged_eg.galtan),
                        n=nrow(merged_eg), type="discriminant"))
        @printf("  Model GAL-TAN   × Expert Economic:  r = %.3f (discriminant, n=%d)\n", r, nrow(merged_eg))
    end

    # 3. Expert galtan vs Model galtan (convergent - already computed, include for matrix)
    galtan_vars = filter(v -> occursin("galtan", v) || occursin("libcon", v), unique(expert.var))
    galtan_expert = filter(row -> row.var in galtan_vars, expert)
    merged_gg = innerjoin(galtan_expert, model_gal, on=[:party, :year])
    if nrow(merged_gg) >= 10
        r = cor(merged_gg.val, merged_gg.galtan)
        push!(results, (model_dim="galtan", expert_dim="galtan",
                        r_pearson=r, r_spearman=corspearman(merged_gg.val, merged_gg.galtan),
                        n=nrow(merged_gg), type="convergent"))
        @printf("  Model GAL-TAN   × Expert GAL-TAN:   r = %.3f (convergent, n=%d)\n", r, nrow(merged_gg))
    end

    # 4. Expert galtan vs Model economic (discriminant)
    merged_ge = innerjoin(galtan_expert, model_econ, on=[:party, :year])
    if nrow(merged_ge) >= 10
        r = cor(merged_ge.val, merged_ge.economic_lr)
        push!(results, (model_dim="economic_lr", expert_dim="galtan",
                        r_pearson=r, r_spearman=corspearman(merged_ge.val, merged_ge.economic_lr),
                        n=nrow(merged_ge), type="discriminant"))
        @printf("  Model Economic  × Expert GAL-TAN:   r = %.3f (discriminant, n=%d)\n", r, nrow(merged_ge))
    end

    println()
    println("MTMM Matrix:")
    println("                    Expert Economic    Expert GAL-TAN")
    for r in results
        if r.model_dim == "economic_lr" && r.expert_dim == "economic"
            @printf("  Model Economic:      %.3f            ", r.r_pearson)
        end
    end
    for r in results
        if r.model_dim == "economic_lr" && r.expert_dim == "galtan"
            @printf("%.3f\n", r.r_pearson)
        end
    end
    for r in results
        if r.model_dim == "galtan" && r.expert_dim == "economic"
            @printf("  Model GAL-TAN:       %.3f            ", r.r_pearson)
        end
    end
    for r in results
        if r.model_dim == "galtan" && r.expert_dim == "galtan"
            @printf("%.3f\n", r.r_pearson)
        end
    end

    return DataFrame(results)
end

function save_validation_results(results::Vector, output_dir::String="validation";
                                  discriminant::Union{DataFrame, Nothing}=nothing)
    """Save validation results to CSV files"""

    if !isdir(output_dir)
        mkpath(output_dir)
    end

    timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")

    # Summary table
    summary_rows = []
    for r in results
        if r !== nothing
            push!(summary_rows, (
                dimension = r.dimension,
                r_pearson = r.r_pearson,
                r_spearman = r.r_spearman,
                ci_lower = r.ci_lower,
                ci_upper = r.ci_upper,
                mae = r.mae,
                rmse = r.rmse,
                n = r.n
            ))
        end
    end

    if !isempty(summary_rows)
        summary_df = DataFrame(summary_rows)
        summary_file = joinpath(output_dir, "convergent_summary_$timestamp.csv")
        CSV.write(summary_file, summary_df)
        println("\nSaved: $summary_file")
    end

    # By-project tables
    for r in results
        if r !== nothing && hasproperty(r, :by_project) && r.by_project !== nothing
            project_file = joinpath(output_dir, "convergent_$(r.dimension)_by_project_$timestamp.csv")
            CSV.write(project_file, r.by_project)
            println("Saved: $project_file")
        end
    end

    # By-decade tables
    for r in results
        if r !== nothing && hasproperty(r, :by_decade) && r.by_decade !== nothing
            decade_file = joinpath(output_dir, "convergent_$(r.dimension)_by_decade_$timestamp.csv")
            CSV.write(decade_file, r.by_decade)
            println("Saved: $decade_file")
        end
    end

    # Discriminant validity table
    if discriminant !== nothing && nrow(discriminant) > 0
        disc_file = joinpath(output_dir, "discriminant_summary_$timestamp.csv")
        CSV.write(disc_file, discriminant)
        println("Saved: $disc_file")
    end

    return summary_rows
end

function print_claassen_comparison(results::Vector)
    """Print comparison with Claassen (2019) benchmarks"""

    println("\n" * "="^60)
    println("COMPARISON WITH CLAASSEN (2019) BENCHMARKS")
    println("="^60)

    println("\nClaassen's results (mood estimates vs survey data):")
    println("  Pearson r: 0.50-0.57")
    println("  MAE: ~0.06 (6 pp on 0-1 scale)")
    println()

    println("Our target (party positions, should be HIGHER than mood):")
    println("  Pearson r > 0.80 with expert surveys")
    println("  MAE < 0.15 (reasonable measurement error)")
    println()

    println("-"^60)
    @printf("%-15s %8s %8s %8s %8s\n", "Dimension", "r", "Target", "MAE", "Status")
    println("-"^60)

    for r in results
        if r !== nothing
            status = r.r_pearson > 0.80 ? "PASS" : (r.r_pearson > 0.70 ? "OK" : "LOW")
            @printf("%-15s %8.3f %8s %8.3f %8s\n",
                r.dimension, r.r_pearson, "> 0.80", r.mae, status)
        end
    end
    println("-"^60)
end

# Main execution
function main()
    println("="^60)
    println("CONVERGENT VALIDITY: Model vs Expert Surveys")
    println("="^60)
    println("Following Claassen (2019) validation framework")
    println()

    # Load data
    model, model_file = load_model_output()
    expert, _ = load_expert_data()

    println("\nModel output:")
    println("  Rows: $(nrow(model))")
    println("  Columns: $(names(model))")

    # Run validations
    results = []

    push!(results, validate_economic_lr(model, expert))
    push!(results, validate_galtan(model, expert))

    # Run discriminant validity
    discriminant = validate_discriminant(model, expert)

    # Save results
    save_validation_results(results; discriminant=discriminant)

    # Print Claassen comparison
    print_claassen_comparison(results)

    println("\n" * "="^60)
    println("VALIDATION COMPLETE")
    println("="^60)

    return results
end

if abspath(PROGRAM_FILE) == @__FILE__
    main()
end
