Document raw source file reference

.gitignore

@@ -0,0 +1,13 @@
+_local/
+tmp/
+outputs/
+
+data/raw/
+data/staging/
+data/releases/
+
+metadata/release_manifest_*.csv
+metadata/scientific_data_release_*.txt
+
+__pycache__/
+*/__pycache__/

Manifest.toml

@@ -0,0 +1,663 @@
+# This file is machine-generated - editing it directly is not advised
+
+julia_version = "1.11.3"
+manifest_format = "2.0"
+project_hash = "5d7dd35b02e3ab85cb2ccc99d83afe86c825420e"
+
+[[deps.AbstractFFTs]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "d92ad398961a3ed262d8bf04a1a2b8340f915fef"
+uuid = "621f4979-c628-5d54-868e-fcf4e3e8185c"
+version = "1.5.0"
+
+    [deps.AbstractFFTs.extensions]
+    AbstractFFTsChainRulesCoreExt = "ChainRulesCore"
+    AbstractFFTsTestExt = "Test"
+
+    [deps.AbstractFFTs.weakdeps]
+    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+    Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[[deps.AliasTables]]
+deps = ["PtrArrays", "Random"]
+git-tree-sha1 = "9876e1e164b144ca45e9e3198d0b689cadfed9ff"
+uuid = "66dad0bd-aa9a-41b7-9441-69ab47430ed8"
+version = "1.1.3"
+
+[[deps.ArgTools]]
+uuid = "0dad84c5-d112-42e6-8d28-ef12dabb789f"
+version = "1.1.2"
+
+[[deps.Artifacts]]
+uuid = "56f22d72-fd6d-98f1-02f0-08ddc0907c33"
+version = "1.11.0"
+
+[[deps.Base64]]
+uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"
+version = "1.11.0"
+
+[[deps.CSV]]
+deps = ["CodecZlib", "Dates", "FilePathsBase", "InlineStrings", "Mmap", "Parsers", "PooledArrays", "PrecompileTools", "SentinelArrays", "Tables", "Unicode", "WeakRefStrings", "WorkerUtilities"]
+git-tree-sha1 = "8d8e0b0f350b8e1c91420b5e64e5de774c2f0f4d"
+uuid = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+version = "0.10.16"
+
+[[deps.CategoricalArrays]]
+deps = ["Compat", "DataAPI", "Future", "Missings", "Printf", "Requires", "Statistics", "Unicode"]
+git-tree-sha1 = "20ff1463035a170b25eba2ef9823bb9ad51635e4"
+uuid = "324d7699-5711-5eae-9e2f-1d82baa6b597"
+version = "1.1.1"
+
+    [deps.CategoricalArrays.extensions]
+    CategoricalArraysArrowExt = "Arrow"
+    CategoricalArraysJSONExt = "JSON"
+    CategoricalArraysRecipesBaseExt = "RecipesBase"
+    CategoricalArraysSentinelArraysExt = "SentinelArrays"
+    CategoricalArraysStatsBaseExt = "StatsBase"
+    CategoricalArraysStructTypesExt = "StructTypes"
+
+    [deps.CategoricalArrays.weakdeps]
+    Arrow = "69666777-d1a9-59fb-9406-91d4454c9d45"
+    JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+    RecipesBase = "3cdcf5f2-1ef4-517c-9805-6587b60abb01"
+    SentinelArrays = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
+    StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+    StructTypes = "856f2bd8-1eba-4b0a-8007-ebc267875bd4"
+
+[[deps.CodecZlib]]
+deps = ["TranscodingStreams", "Zlib_jll"]
+git-tree-sha1 = "962834c22b66e32aa10f7611c08c8ca4e20749a9"
+uuid = "944b1d66-785c-5afd-91f1-9de20f533193"
+version = "0.7.8"
+
+[[deps.Compat]]
+deps = ["TOML", "UUIDs"]
+git-tree-sha1 = "9d8a54ce4b17aa5bdce0ea5c34bc5e7c340d16ad"
+uuid = "34da2185-b29b-5c13-b0c7-acf172513d20"
+version = "4.18.1"
+weakdeps = ["Dates", "LinearAlgebra"]
+
+    [deps.Compat.extensions]
+    CompatLinearAlgebraExt = "LinearAlgebra"
+
+[[deps.CompatHelperLocal]]
+deps = ["Pkg"]
+git-tree-sha1 = "f239f702063ca2fc50844c6cf27b4d497652b474"
+uuid = "5224ae11-6099-4aaa-941d-3aab004bd678"
+version = "0.1.29"
+
+[[deps.CompilerSupportLibraries_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "e66e0078-7015-5450-92f7-15fbd957f2ae"
+version = "1.1.1+0"
+
+[[deps.Crayons]]
+git-tree-sha1 = "249fe38abf76d48563e2f4556bebd215aa317e15"
+uuid = "a8cc5b0e-0ffa-5ad4-8c14-923d3ee1735f"
+version = "4.1.1"
+
+[[deps.DataAPI]]
+git-tree-sha1 = "abe83f3a2f1b857aac70ef8b269080af17764bbe"
+uuid = "9a962f9c-6df0-11e9-0e5d-c546b8b5ee8a"
+version = "1.16.0"
+
+[[deps.DataFrames]]
+deps = ["Compat", "DataAPI", "DataStructures", "Future", "InlineStrings", "InvertedIndices", "IteratorInterfaceExtensions", "LinearAlgebra", "Markdown", "Missings", "PooledArrays", "PrecompileTools", "PrettyTables", "Printf", "Random", "Reexport", "SentinelArrays", "SortingAlgorithms", "Statistics", "TableTraits", "Tables", "Unicode"]
+git-tree-sha1 = "5fab31e2e01e70ad66e3e24c968c264d1cf166d6"
+uuid = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+version = "1.8.2"
+
+[[deps.DataStructures]]
+deps = ["OrderedCollections"]
+git-tree-sha1 = "6fb53a69613a0b2b68a0d12671717d307ab8b24e"
+uuid = "864edb3b-99cc-5e75-8d2d-829cb0a9cfe8"
+version = "0.19.5"
+
+[[deps.DataValueInterfaces]]
+git-tree-sha1 = "bfc1187b79289637fa0ef6d4436ebdfe6905cbd6"
+uuid = "e2d170a0-9d28-54be-80f0-106bbe20a464"
+version = "1.0.0"
+
+[[deps.Dates]]
+deps = ["Printf"]
+uuid = "ade2ca70-3891-5945-98fb-dc099432e06a"
+version = "1.11.0"
+
+[[deps.DelimitedFiles]]
+deps = ["Mmap"]
+git-tree-sha1 = "9e2f36d3c96a820c678f2f1f1782582fcf685bae"
+uuid = "8bb1440f-4735-579b-a4ab-409b98df4dab"
+version = "1.9.1"
+
+[[deps.Distributed]]
+deps = ["Random", "Serialization", "Sockets"]
+uuid = "8ba89e20-285c-5b6f-9357-94700520ee1b"
+version = "1.11.0"
+
+[[deps.DocStringExtensions]]
+git-tree-sha1 = "7442a5dfe1ebb773c29cc2962a8980f47221d76c"
+uuid = "ffbed154-4ef7-542d-bbb7-c09d3a79fcae"
+version = "0.9.5"
+
+[[deps.Downloads]]
+deps = ["ArgTools", "FileWatching", "LibCURL", "NetworkOptions"]
+uuid = "f43a241f-c20a-4ad4-852c-f6b1247861c6"
+version = "1.6.0"
+
+[[deps.FFTW]]
+deps = ["AbstractFFTs", "FFTW_jll", "Libdl", "LinearAlgebra", "MKL_jll", "Preferences", "Reexport"]
+git-tree-sha1 = "97f08406df914023af55ade2f843c39e99c5d969"
+uuid = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
+version = "1.10.0"
+
+[[deps.FFTW_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl"]
+git-tree-sha1 = "6866aec60ef98e3164cd8d6855225684207e9dff"
+uuid = "f5851436-0d7a-5f13-b9de-f02708fd171a"
+version = "3.3.12+0"
+
+[[deps.FilePathsBase]]
+deps = ["Compat", "Dates"]
+git-tree-sha1 = "3bab2c5aa25e7840a4b065805c0cdfc01f3068d2"
+uuid = "48062228-2e41-5def-b9a4-89aafe57970f"
+version = "0.9.24"
+weakdeps = ["Mmap", "Test"]
+
+    [deps.FilePathsBase.extensions]
+    FilePathsBaseMmapExt = "Mmap"
+    FilePathsBaseTestExt = "Test"
+
+[[deps.FileWatching]]
+uuid = "7b1f6079-737a-58dc-b8bc-7a2ca5c1b5ee"
+version = "1.11.0"
+
+[[deps.Future]]
+deps = ["Random"]
+uuid = "9fa8497b-333b-5362-9e8d-4d0656e87820"
+version = "1.11.0"
+
+[[deps.HypergeometricFunctions]]
+deps = ["LinearAlgebra", "OpenLibm_jll", "SpecialFunctions"]
+git-tree-sha1 = "68c173f4f449de5b438ee67ed0c9c748dc31a2ec"
+uuid = "34004b35-14d8-5ef3-9330-4cdb6864b03a"
+version = "0.3.28"
+
+[[deps.InlineStrings]]
+git-tree-sha1 = "8f3d257792a522b4601c24a577954b0a8cd7334d"
+uuid = "842dd82b-1e85-43dc-bf29-5d0ee9dffc48"
+version = "1.4.5"
+
+    [deps.InlineStrings.extensions]
+    ArrowTypesExt = "ArrowTypes"
+    ParsersExt = "Parsers"
+
+    [deps.InlineStrings.weakdeps]
+    ArrowTypes = "31f734f8-188a-4ce0-8406-c8a06bd891cd"
+    Parsers = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
+
+[[deps.IntelOpenMP_jll]]
+deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"]
+git-tree-sha1 = "ec1debd61c300961f98064cfb21287613ad7f303"
+uuid = "1d5cc7b8-4909-519e-a0f8-d0f5ad9712d0"
+version = "2025.2.0+0"
+
+[[deps.InteractiveUtils]]
+deps = ["Markdown"]
+uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+version = "1.11.0"
+
+[[deps.InvertedIndices]]
+git-tree-sha1 = "6da3c4316095de0f5ee2ebd875df8721e7e0bdbe"
+uuid = "41ab1584-1d38-5bbf-9106-f11c6c58b48f"
+version = "1.3.1"
+
+[[deps.IrrationalConstants]]
+git-tree-sha1 = "b2d91fe939cae05960e760110b328288867b5758"
+uuid = "92d709cd-6900-40b7-9082-c6be49f344b6"
+version = "0.2.6"
+
+[[deps.IteratorInterfaceExtensions]]
+git-tree-sha1 = "a3f24677c21f5bbe9d2a714f95dcd58337fb2856"
+uuid = "82899510-4779-5014-852e-03e436cf321d"
+version = "1.0.0"
+
+[[deps.JLLWrappers]]
+deps = ["Artifacts", "Preferences"]
+git-tree-sha1 = "7204148362dafe5fe6a273f855b8ccbe4df8173e"
+uuid = "692b3bcd-3c85-4b1f-b108-f13ce0eb3210"
+version = "1.8.0"
+
+[[deps.JSON]]
+deps = ["Dates", "Mmap", "Parsers", "Unicode"]
+git-tree-sha1 = "31e996f0a15c7b280ba9f76636b3ff9e2ae58c9a"
+uuid = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+version = "0.21.4"
+
+[[deps.LaTeXStrings]]
+git-tree-sha1 = "dda21b8cbd6a6c40d9d02a73230f9d70fed6918c"
+uuid = "b964fa9f-0449-5b57-a5c2-d3ea65f4040f"
+version = "1.4.0"
+
+[[deps.LazyArtifacts]]
+deps = ["Artifacts", "Pkg"]
+uuid = "4af54fe1-eca0-43a8-85a7-787d91b784e3"
+version = "1.11.0"
+
+[[deps.LibCURL]]
+deps = ["LibCURL_jll", "MozillaCACerts_jll"]
+uuid = "b27032c2-a3e7-50c8-80cd-2d36dbcbfd21"
+version = "0.6.4"
+
+[[deps.LibCURL_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll", "Zlib_jll", "nghttp2_jll"]
+uuid = "deac9b47-8bc7-5906-a0fe-35ac56dc84c0"
+version = "8.6.0+0"
+
+[[deps.LibGit2]]
+deps = ["Base64", "LibGit2_jll", "NetworkOptions", "Printf", "SHA"]
+uuid = "76f85450-5226-5b5a-8eaa-529ad045b433"
+version = "1.11.0"
+
+[[deps.LibGit2_jll]]
+deps = ["Artifacts", "LibSSH2_jll", "Libdl", "MbedTLS_jll"]
+uuid = "e37daf67-58a4-590a-8e99-b0245dd2ffc5"
+version = "1.7.2+0"
+
+[[deps.LibSSH2_jll]]
+deps = ["Artifacts", "Libdl", "MbedTLS_jll"]
+uuid = "29816b5a-b9ab-546f-933c-edad1886dfa8"
+version = "1.11.0+1"
+
+[[deps.Libdl]]
+uuid = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+version = "1.11.0"
+
+[[deps.LinearAlgebra]]
+deps = ["Libdl", "OpenBLAS_jll", "libblastrampoline_jll"]
+uuid = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+version = "1.11.0"
+
+[[deps.LogExpFunctions]]
+deps = ["DocStringExtensions", "IrrationalConstants", "LinearAlgebra"]
+git-tree-sha1 = "bba2d9aa057d8f126415de240573e86a8f39d2a1"
+uuid = "2ab3a3ac-af41-5b50-aa03-7779005ae688"
+version = "1.0.1"
+
+    [deps.LogExpFunctions.extensions]
+    LogExpFunctionsChainRulesCoreExt = "ChainRulesCore"
+    LogExpFunctionsChangesOfVariablesExt = "ChangesOfVariables"
+    LogExpFunctionsInverseFunctionsExt = "InverseFunctions"
+
+    [deps.LogExpFunctions.weakdeps]
+    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+    ChangesOfVariables = "9e997f8a-9a97-42d5-a9f1-ce6bfc15e2c0"
+    InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112"
+
+[[deps.Logging]]
+uuid = "56ddb016-857b-54e1-b83d-db4d58db5568"
+version = "1.11.0"
+
+[[deps.MKL_jll]]
+deps = ["Artifacts", "IntelOpenMP_jll", "JLLWrappers", "LazyArtifacts", "Libdl", "oneTBB_jll"]
+git-tree-sha1 = "282cadc186e7b2ae0eeadbd7a4dffed4196ae2aa"
+uuid = "856f044c-d86e-5d09-b602-aeab76dc8ba7"
+version = "2025.2.0+0"
+
+[[deps.Markdown]]
+deps = ["Base64"]
+uuid = "d6f4376e-aef5-505a-96c1-9c027394607a"
+version = "1.11.0"
+
+[[deps.MbedTLS_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "c8ffd9c3-330d-5841-b78e-0817d7145fa1"
+version = "2.28.6+0"
+
+[[deps.Missings]]
+deps = ["DataAPI"]
+git-tree-sha1 = "ec4f7fbeab05d7747bdf98eb74d130a2a2ed298d"
+uuid = "e1d29d7a-bbdc-5cf2-9ac0-f12de2c33e28"
+version = "1.2.0"
+
+[[deps.Mmap]]
+uuid = "a63ad114-7e13-5084-954f-fe012c677804"
+version = "1.11.0"
+
+[[deps.MozillaCACerts_jll]]
+uuid = "14a3606d-f60d-562e-9121-12d972cd8159"
+version = "2023.12.12"
+
+[[deps.NamedTupleTools]]
+git-tree-sha1 = "90914795fc59df44120fe3fff6742bb0d7adb1d0"
+uuid = "d9ec5142-1e00-5aa0-9d6a-321866360f50"
+version = "0.14.3"
+
+[[deps.NetworkOptions]]
+uuid = "ca575930-c2e3-43a9-ace4-1e988b2c1908"
+version = "1.2.0"
+
+[[deps.OpenBLAS_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "Libdl"]
+uuid = "4536629a-c528-5b80-bd46-f80d51c5b363"
+version = "0.3.27+1"
+
+[[deps.OpenLibm_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "05823500-19ac-5b8b-9628-191a04bc5112"
+version = "0.8.1+2"
+
+[[deps.OpenSpecFun_jll]]
+deps = ["Artifacts", "CompilerSupportLibraries_jll", "JLLWrappers", "Libdl"]
+git-tree-sha1 = "1346c9208249809840c91b26703912dff463d335"
+uuid = "efe28fd5-8261-553b-a9e1-b2916fc3738e"
+version = "0.5.6+0"
+
+[[deps.OrderedCollections]]
+git-tree-sha1 = "94ba93778373a53bfd5a0caaf7d809c445292ff4"
+uuid = "bac558e1-5e72-5ebc-8fee-abe8a469f55d"
+version = "1.8.2"
+
+[[deps.Parameters]]
+deps = ["OrderedCollections", "UnPack"]
+git-tree-sha1 = "34c0e9ad262e5f7fc75b10a9952ca7692cfc5fbe"
+uuid = "d96e819e-fc66-5662-9728-84c9c7592b0a"
+version = "0.12.3"
+
+[[deps.Parsers]]
+deps = ["Dates", "PrecompileTools", "UUIDs"]
+git-tree-sha1 = "468dbe2b510c876dc091b2c74ed52c7c34f48b9b"
+uuid = "69de0a69-1ddd-5017-9359-2bf0b02dc9f0"
+version = "2.8.5"
+
+[[deps.Pkg]]
+deps = ["Artifacts", "Dates", "Downloads", "FileWatching", "LibGit2", "Libdl", "Logging", "Markdown", "Printf", "Random", "SHA", "TOML", "Tar", "UUIDs", "p7zip_jll"]
+uuid = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+version = "1.11.0"
+weakdeps = ["REPL"]
+
+    [deps.Pkg.extensions]
+    REPLExt = "REPL"
+
+[[deps.PooledArrays]]
+deps = ["DataAPI", "Future"]
+git-tree-sha1 = "36d8b4b899628fb92c2749eb488d884a926614d3"
+uuid = "2dfb63ee-cc39-5dd5-95bd-886bf059d720"
+version = "1.4.3"
+
+[[deps.PrecompileTools]]
+deps = ["Preferences"]
+git-tree-sha1 = "5aa36f7049a63a1528fe8f7c3f2113413ffd4e1f"
+uuid = "aea7be01-6a6a-4083-8856-8a6e6704d82a"
+version = "1.2.1"
+
+[[deps.Preferences]]
+deps = ["TOML"]
+git-tree-sha1 = "8b770b60760d4451834fe79dd483e318eee709c4"
+uuid = "21216c6a-2e73-6563-6e65-726566657250"
+version = "1.5.2"
+
+[[deps.PrettyTables]]
+deps = ["Crayons", "LaTeXStrings", "Markdown", "PrecompileTools", "Printf", "REPL", "Reexport", "StringManipulation", "Tables"]
+git-tree-sha1 = "624de6279ab7d94fc9f672f0068107eb6619732c"
+uuid = "08abe8d2-0d0c-5749-adfa-8a2ac140af0d"
+version = "3.3.2"
+
+    [deps.PrettyTables.extensions]
+    PrettyTablesTypstryExt = "Typstry"
+
+    [deps.PrettyTables.weakdeps]
+    Typstry = "f0ed7684-a786-439e-b1e3-3b82803b501e"
+
+[[deps.Printf]]
+deps = ["Unicode"]
+uuid = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+version = "1.11.0"
+
+[[deps.ProgressMeter]]
+deps = ["Distributed", "Printf"]
+git-tree-sha1 = "fbb92c6c56b34e1a2c4c36058f68f332bec840e7"
+uuid = "92933f4c-e287-5a05-a399-4b506db050ca"
+version = "1.11.0"
+
+[[deps.PtrArrays]]
+git-tree-sha1 = "4fbbafbc6251b883f4d2705356f3641f3652a7fe"
+uuid = "43287f4e-b6f4-7ad1-bb20-aadabca52c3d"
+version = "1.4.0"
+
+[[deps.REPL]]
+deps = ["InteractiveUtils", "Markdown", "Sockets", "StyledStrings", "Unicode"]
+uuid = "3fa0cd96-eef1-5676-8a61-b3b8758bbffb"
+version = "1.11.0"
+
+[[deps.Random]]
+deps = ["SHA"]
+uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+version = "1.11.0"
+
+[[deps.Reexport]]
+git-tree-sha1 = "45e428421666073eab6f2da5c9d310d99bb12f9b"
+uuid = "189a3867-3050-52da-a836-e630ba90ab69"
+version = "1.2.2"
+
+[[deps.Requires]]
+deps = ["UUIDs"]
+git-tree-sha1 = "62389eeff14780bfe55195b7204c0d8738436d64"
+uuid = "ae029012-a4dd-5104-9daa-d747884805df"
+version = "1.3.1"
+
+[[deps.Rmath]]
+deps = ["Random", "Rmath_jll"]
+git-tree-sha1 = "5b3d50eb374cea306873b371d3f8d3915a018f0b"
+uuid = "79098fc4-a85e-5d69-aa6a-4863f24498fa"
+version = "0.9.0"
+
+[[deps.Rmath_jll]]
+deps = ["Artifacts", "JLLWrappers", "Libdl"]
+git-tree-sha1 = "58cdd8fb2201a6267e1db87ff148dd6c1dbd8ad8"
+uuid = "f50d1b31-88e8-58de-be2c-1cc44531875f"
+version = "0.5.1+0"
+
+[[deps.SHA]]
+uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce"
+version = "0.7.0"
+
+[[deps.SentinelArrays]]
+deps = ["Dates", "Random"]
+git-tree-sha1 = "084c47c7c5ce5cfecefa0a98dff69eb3646b5a80"
+uuid = "91c51154-3ec4-41a3-a24f-3f23e20d615c"
+version = "1.4.10"
+
+[[deps.Serialization]]
+uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b"
+version = "1.11.0"
+
+[[deps.Sockets]]
+uuid = "6462fe0b-24de-5631-8697-dd941f90decc"
+version = "1.11.0"
+
+[[deps.SortingAlgorithms]]
+deps = ["DataStructures"]
+git-tree-sha1 = "64d974c2e6fdf07f8155b5b2ca2ffa9069b608d9"
+uuid = "a2af1166-a08f-5f64-846c-94a0d3cef48c"
+version = "1.2.2"
+
+[[deps.SparseArrays]]
+deps = ["Libdl", "LinearAlgebra", "Random", "Serialization", "SuiteSparse_jll"]
+uuid = "2f01184e-e22b-5df5-ae63-d93ebab69eaf"
+version = "1.11.0"
+
+[[deps.SpecialFunctions]]
+deps = ["IrrationalConstants", "LogExpFunctions", "OpenLibm_jll", "OpenSpecFun_jll"]
+git-tree-sha1 = "6547cbdd8ce32efba0d21c5a40fa96d1a3548f9f"
+uuid = "276daf66-3868-5448-9aa4-cd146d93841b"
+version = "2.8.0"
+
+    [deps.SpecialFunctions.extensions]
+    SpecialFunctionsChainRulesCoreExt = "ChainRulesCore"
+
+    [deps.SpecialFunctions.weakdeps]
+    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+
+[[deps.StanBase]]
+deps = ["CSV", "DataFrames", "DelimitedFiles", "Distributed", "DocStringExtensions", "JSON", "NamedTupleTools", "OrderedCollections", "Parameters", "Random", "Unicode"]
+git-tree-sha1 = "fac478e96c6a32f08af58f709b07f5ee722b3140"
+uuid = "d0ee94f6-a23d-54aa-bbe9-7f572d6da7f5"
+version = "4.12.4"
+
+[[deps.StanSample]]
+deps = ["CSV", "CompatHelperLocal", "DataFrames", "DelimitedFiles", "DocStringExtensions", "JSON", "LazyArtifacts", "NamedTupleTools", "OrderedCollections", "Parameters", "Random", "Reexport", "Requires", "Serialization", "StanBase", "TableOperations", "Tables", "Unicode"]
+git-tree-sha1 = "0e6f7c729879a416fd7b27cfd81b2b1f6e08586e"
+uuid = "c1514b29-d3a0-5178-b312-660c88baa699"
+version = "7.10.3"
+
+    [deps.StanSample.extensions]
+    AxisKeysExt = "AxisKeys"
+    InferenceObjectsExt = "InferenceObjects"
+    MCMCChainsExt = "MCMCChains"
+    MonteCarloMeasurementsExt = "MonteCarloMeasurements"
+
+    [deps.StanSample.weakdeps]
+    AxisKeys = "94b1ba4f-4ee9-5380-92f1-94cde586c3c5"
+    InferenceObjects = "b5cf5a8d-e756-4ee3-b014-01d49d192c00"
+    MCMCChains = "c7f686f2-ff18-58e9-bc7b-31028e88f75d"
+    MonteCarloMeasurements = "0987c9cc-fe09-11e8-30f0-b96dd679fdca"
+
+[[deps.Statistics]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "ae3bb1eb3bba077cd276bc5cfc337cc65c3075c0"
+uuid = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+version = "1.11.1"
+weakdeps = ["SparseArrays"]
+
+    [deps.Statistics.extensions]
+    SparseArraysExt = ["SparseArrays"]
+
+[[deps.StatsAPI]]
+deps = ["LinearAlgebra"]
+git-tree-sha1 = "178ed29fd5b2a2cfc3bd31c13375ae925623ff36"
+uuid = "82ae8749-77ed-4fe6-ae5f-f523153014b0"
+version = "1.8.0"
+
+[[deps.StatsBase]]
+deps = ["AliasTables", "DataAPI", "DataStructures", "IrrationalConstants", "LinearAlgebra", "LogExpFunctions", "Missings", "Printf", "Random", "SortingAlgorithms", "SparseArrays", "Statistics", "StatsAPI"]
+git-tree-sha1 = "c6f18e5a52a176a383f6f6c635e0f81feed1d6d4"
+uuid = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+version = "0.34.11"
+
+[[deps.StatsFuns]]
+deps = ["HypergeometricFunctions", "IrrationalConstants", "LogExpFunctions", "Reexport", "Rmath", "SpecialFunctions"]
+git-tree-sha1 = "3f4e1d24289cd974e089c617b1472311a2b1feab"
+uuid = "4c63d2b9-4356-54db-8cca-17b64c39e42c"
+version = "2.1.0"
+
+    [deps.StatsFuns.extensions]
+    StatsFunsChainRulesCoreExt = "ChainRulesCore"
+    StatsFunsInverseFunctionsExt = "InverseFunctions"
+
+    [deps.StatsFuns.weakdeps]
+    ChainRulesCore = "d360d2e6-b24c-11e9-a2a3-2a2ae2dbcce4"
+    InverseFunctions = "3587e190-3f89-42d0-90ee-14403ec27112"
+
+[[deps.StringManipulation]]
+deps = ["PrecompileTools"]
+git-tree-sha1 = "d05693d339e37d6ab134c5ab53c29fce5ee5d7d5"
+uuid = "892a3eda-7b42-436c-8928-eab12a02cf0e"
+version = "0.4.4"
+
+[[deps.StyledStrings]]
+uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b"
+version = "1.11.0"
+
+[[deps.SuiteSparse_jll]]
+deps = ["Artifacts", "Libdl", "libblastrampoline_jll"]
+uuid = "bea87d4a-7f5b-5778-9afe-8cc45184846c"
+version = "7.7.0+0"
+
+[[deps.TOML]]
+deps = ["Dates"]
+uuid = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
+version = "1.0.3"
+
+[[deps.TableOperations]]
+deps = ["SentinelArrays", "Tables", "Test"]
+git-tree-sha1 = "e383c87cf2a1dc41fa30c093b2a19877c83e1bc1"
+uuid = "ab02a1b2-a7df-11e8-156e-fb1833f50b87"
+version = "1.2.0"
+
+[[deps.TableTraits]]
+deps = ["IteratorInterfaceExtensions"]
+git-tree-sha1 = "c06b2f539df1c6efa794486abfb6ed2022561a39"
+uuid = "3783bdb8-4a98-5b6b-af9a-565f29a5fe9c"
+version = "1.0.1"
+
+[[deps.Tables]]
+deps = ["DataAPI", "DataValueInterfaces", "IteratorInterfaceExtensions", "OrderedCollections", "TableTraits"]
+git-tree-sha1 = "f2c1efbc8f3a609aadf318094f8fc5204bdaf344"
+uuid = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
+version = "1.12.1"
+
+[[deps.Tar]]
+deps = ["ArgTools", "SHA"]
+uuid = "a4e569a6-e804-4fa4-b0f3-eef7a1d5b13e"
+version = "1.10.0"
+
+[[deps.Test]]
+deps = ["InteractiveUtils", "Logging", "Random", "Serialization"]
+uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+version = "1.11.0"
+
+[[deps.TranscodingStreams]]
+git-tree-sha1 = "0c45878dcfdcfa8480052b6ab162cdd138781742"
+uuid = "3bb67fe8-82b1-5028-8e26-92a6c54297fa"
+version = "0.11.3"
+
+[[deps.UUIDs]]
+deps = ["Random", "SHA"]
+uuid = "cf7118a7-6976-5b1a-9a39-7adc72f591a4"
+version = "1.11.0"
+
+[[deps.UnPack]]
+git-tree-sha1 = "387c1f73762231e86e0c9c5443ce3b4a0a9a0c2b"
+uuid = "3a884ed6-31ef-47d7-9d2a-63182c4928ed"
+version = "1.0.2"
+
+[[deps.Unicode]]
+uuid = "4ec0a83e-493e-50e2-b9ac-8f72acf5a8f5"
+version = "1.11.0"
+
+[[deps.WeakRefStrings]]
+deps = ["DataAPI", "InlineStrings", "Parsers"]
+git-tree-sha1 = "0716e01c3b40413de5dedbc9c5c69f27cddfddfc"
+uuid = "ea10d353-3f73-51f8-a26c-33c1cb351aa5"
+version = "1.4.3"
+
+[[deps.WorkerUtilities]]
+git-tree-sha1 = "cd1659ba0d57b71a464a29e64dbc67cfe83d54e7"
+uuid = "76eceee3-57b5-4d4a-8e66-0e911cebbf60"
+version = "1.6.1"
+
+[[deps.Zlib_jll]]
+deps = ["Libdl"]
+uuid = "83775a58-1f1d-513f-b197-d71354ab007a"
+version = "1.2.13+1"
+
+[[deps.libblastrampoline_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850b90-86db-534c-a0d3-1478176c7d93"
+version = "5.11.0+0"
+
+[[deps.nghttp2_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "8e850ede-7688-5339-a07c-302acd2aaf8d"
+version = "1.59.0+0"
+
+[[deps.oneTBB_jll]]
+deps = ["Artifacts", "JLLWrappers", "LazyArtifacts", "Libdl"]
+git-tree-sha1 = "da8c1f6eee04831f14edcfa5dae611d309807e57"
+uuid = "1317d2d5-d96f-522e-a858-c73665f53c3e"
+version = "2022.3.0+0"
+
+[[deps.p7zip_jll]]
+deps = ["Artifacts", "Libdl"]
+uuid = "3f19e933-33d8-53b3-aaab-bd5110c3b7a0"
+version = "17.4.0+2"

Project.toml

@@ -0,0 +1,10 @@
+[deps]
+CSV = "336ed68f-0bac-5ca0-87d4-7b16caf5d00b"
+CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+FFTW = "7a1cc6ca-52ef-59f5-83cd-3a7055c09341"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+ProgressMeter = "92933f4c-e287-5a05-a399-4b506db050ca"
+StanSample = "c1514b29-d3a0-5178-b312-660c88baa699"
+StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
+StatsFuns = "4c63d2b9-4356-54db-8cca-17b64c39e42c"

README.md

@@ -0,0 +1,64 @@
+# party2d
+
+Code and processed model inputs for generating two-dimensional party-position estimates from text and expert data. The model combines manifesto and media text indicators with expert survey placements in a Bayesian dynamic item-response framework.
+
+## Repository contents
+
+- `src/r/` — scripts that prepare processed model inputs from source datasets.
+- `src/julia/` — Stan data preparation, model fitting, post-estimation, enrichment, and validation.
+- `models/` — Stan model specification.
+- `data/` — processed party-level inputs used by the model.
+- `metadata/` — data dictionary and source-support documentation.
+- `docs/` — raw data source documentation, coding decisions, and operational notes.
+
+Processed inputs needed by the model are included in `data/` so the estimation step can be reproduced from the model-ready data.
+
+## Running the pipeline
+
+Run the full workflow with:
+
+```bash
+bash run_estimation.sh full
+```
+
+This executes the numbered workflow scripts:
+
+```bash
+bash scripts/01_prepare_data.sh
+bash scripts/02_fit_model.sh
+bash scripts/03_extract_estimates.sh
+bash scripts/04_enrich_estimates.sh
+```
+
+The numbered scripts can also be run manually in that order. `scripts/05_validate_estimates.sh` runs validation checks after estimates have been generated.
+
+The Bayesian model is computationally expensive. The production run used 4 cores on an AMD Ryzen 9 7945HX and took 60,372 seconds, approximately 16 hours 46 minutes.
+
+If model output is already available, rebuild estimates without refitting Stan:
+
+```bash
+bash run_estimation.sh reuse
+```
+
+`reuse` reruns source-data processing, post-estimation, and enrichment while skipping the Stan fitting step.
+
+To check the local setup without fitting the model, run:
+
+```bash
+bash run_estimation.sh dry-run
+```
+
+## Data inputs
+
+The model-ready inputs are included under `data/`.
+
+Original raw source files are not redistributed. See `docs/RAW_DATA_SOURCES.md` for the list of original data sources, access information, and expected local filenames for regenerating the processed inputs.
+
+## Output variables
+
+The two position variables are scaled from 0 to 1:
+
+- `economic_lr`: economic left to economic right.
+- `galtan`: cosmopolitan/socially liberal to traditionalist/nationalist.
+
+Column definitions are in `metadata/data_dictionary.csv`.

data/README.md

@@ -0,0 +1,7 @@
+# Data directory
+
+This directory contains processed, model-ready party-level inputs used by the estimation pipeline.
+
+Original raw source files are not stored here. To regenerate the processed inputs, place raw files in a local directory and set `PARTY2D_RAW_DATA_DIR`; see `../docs/RAW_DATA_SOURCES.md`.
+
+Generated outputs and temporary staging files are ignored by git.

data/morgan_data.csv

@@ -0,0 +1,150 @@
+country,partyfacts_id,period,year_start,year_end,party_abbrev,party_name,lr_morgan,lr_morgan_se,n_surveys
+BEL,302,1919-1939,1919,1939,COMM,Communist Party,0.005,0.01,11
+BEL,2289,1919-1939,1919,1939,POB,Social Democratic Party,0.2511,0.016522821682861448,11
+BEL,1892,1919-1939,1919,1939,CATH,Catholic Party,0.6785,0.02948780949970528,11
+BEL,275,1919-1939,1919,1939,LIB,Liberal Party,0.7884,0.026152036249592493,10
+BEL,1834,1919-1939,1919,1939,FNAT,Flemish Nationalist parties,0.8078,0.0359703034081272,11
+BEL,1829,1919-1939,1919,1939,REX,Rexist,0.9872,0.01,10
+BEL,302,1945-1973,1945,1973,COMM,Communist Party,0.005,0.01,17
+BEL,480,1945-1973,1945,1973,PSB,Socialist Party,0.3193,0.01273312031440748,17
+BEL,281,1945-1973,1945,1973,RW,Rassemblement Wallon,0.5365,0.024325,16
+BEL,405,1945-1973,1945,1973,PSC,Christian Social Party,0.7090000000000001,0.01598309768989434,17
+BEL,1333,1945-1973,1945,1973,FDF,Front Democratique Wallon,0.7189,0.033591184067532115,17
+BEL,1424,1945-1973,1945,1973,VOLK,Volksunie,0.8321,0.01610436550241251,17
+BEL,49,1945-1973,1945,1973,PLP,Liberal Party,0.995,0.01,17
+DEU,1135,1919-1930,1919,1930,KPD,Communist Party,0.005,0.01,15
+DEU,383,1919-1930,1919,1930,SDAP,Social Democratic Party,0.25579999999999997,0.011928788706318843,15
+DEU,1826,1919-1930,1919,1930,DDP,Democratic Party,0.4557,0.012367726818889016,15
+DEU,1798,1919-1930,1919,1930,DZP,Center Party,0.5125,0.01639562949894473,15
+DEU,1825,1919-1930,1919,1930,BVP,Bavarian People's Party,0.6317,0.01697508869972809,11
+DEU,1827,1919-1930,1919,1930,DVP,German People's Party,0.6517000000000001,0.012367726818889016,15
+DEU,1828,1919-1930,1919,1930,RDMW,Economic Party,0.7064,0.017879622733461383,11
+DEU,1831,1919-1930,1919,1930,LVP,Rural People's Party,0.8035,0.021766666666666667,9
+DEU,1830,1919-1930,1919,1930,DNVP,German National People's Party,0.8817,0.01,15
+DEU,1893,1919-1930,1919,1930,NAZI,National Socialist Party,0.995,0.01,15
+DNK,379,1919-1939,1919,1939,SOCd,Social Democrats,0.005,0.01,19
+DNK,1507,1919-1939,1919,1939,RAD,Radicals,0.3394,0.010599006904819953,19
+DNK,1204,1919-1939,1919,1939,LIB,Liberals,0.7187,0.018330317136257882,19
+DNK,536,1919-1939,1919,1939,CONS,Conservatives,0.995,0.01,19
+DNK,277,1945-1973,1945,1973,COMM,Communists,0.005,0.01,22
+DNK,1136,1945-1973,1945,1973,LS,Left Socialists,0.065,0.01,22
+DNK,329,1945-1973,1945,1973,SOCL,Socialists (SF),0.1954,0.012642802479887697,22
+DNK,379,1945-1973,1945,1973,SOCd,Social Democrats,0.41490000000000005,0.015329131505968391,22
+DNK,1507,1945-1973,1945,1973,RAD,Radicals,0.6185,0.01328240462895453,22
+DNK,1118,1945-1973,1945,1973,LC,Liberal Center,0.679,0.019492576934709162,18
+DNK,1204,1945-1973,1945,1973,LIB,Liberals,0.8198000000000001,0.013772766276572434,22
+DNK,1134,1945-1973,1945,1973,JUST,Justice Party,0.8408,0.03245126662986135,17
+DNK,536,1945-1973,1945,1973,CONS,Conservatives,0.995,0.01,22
+FIN,1904,1919-1939,1919,1939,SKDL,Communist,0.005,0.01,12
+FIN,1303,1919-1939,1919,1939,SOCd,Social Democrats,0.2492,0.01,12
+FIN,1862,1919-1939,1919,1939,PROG,National Progressives,0.575,0.013509996299037243,12
+FIN,901,1919-1939,1919,1939,AGR,Agrarian Union,0.5932,0.012874911002928655,12
+FIN,1229,1919-1939,1919,1939,SWPP,Swedish People's Party,0.6779000000000001,0.01896595634287921,12
+FIN,495,1919-1939,1919,1939,CONS,Finnish Conservative Party,0.9196,0.011258330249197703,12
+FIN,725,1919-1939,1919,1939,NPF,Finnish Patriotic Movement,0.995,0.01,12
+FIN,1096,1945-1973,1945,1973,PDEM,Democratic League,0.005,0.01,15
+FIN,1164,1945-1973,1945,1973,SDWS,Workers Smallholders SD League,0.2115,0.017376785279983945,15
+FIN,1303,1945-1973,1945,1973,SOCd,Social Democrats,0.25780000000000003,0.01,15
+FIN,901,1945-1973,1945,1973,CENT,Center Party,0.5776,0.012175669307143778,13
+FIN,1689,1945-1973,1945,1973,FRP,Finnish Rural Party,0.6411,0.046899901590843125,13
+FIN,249,1945-1973,1945,1973,LIB,Finnish People Party/Liberal League,0.7034,0.014200938936093862,15
+FIN,1229,1945-1973,1945,1973,SWPP,Swedish People's Party,0.7975,0.01608579083124814,15
+FIN,495,1945-1973,1945,1973,CONS,Finnish Conservative Party,0.9854999999999999,0.01,15
+FRA,1251,1946-1958,1946,1958,PCF,Communist Party,0.005,0.01,23
+FRA,1478,1946-1958,1946,1958,SFIO,Socialist Party,0.2365,0.01,23
+FRA,737,1946-1958,1946,1958,MRP,Popular Republican Movement,0.43119999999999997,0.014033020066041133,23
+FRA,14,1946-1958,1946,1958,RDA,Radical Party,0.4337,0.015048737488573584,20
+FRA,2470,1946-1958,1946,1958,UDSR,Democratic Socialist Union,0.4462,0.014936934089698595,20
+FRA,1083,1946-1958,1946,1958,RPF,Rally of the French People,0.7,0.01511729501913792,23
+FRA,2689,1946-1958,1946,1958,AR,Republican Action,0.7293000000000001,0.01,20
+FRA,3181,1946-1958,1946,1958,ARS,Republican Social Action,0.7737999999999999,0.017075,16
+FRA,1246,1946-1958,1946,1958,RI,Independent Republicans,0.8262,0.01846796657658022,19
+FRA,1629,1946-1958,1946,1958,CNIP,National Center of Independents,0.835,0.011700202427398648,19
+FRA,1898,1946-1958,1946,1958,AP,Popular Action,0.8540000000000001,0.013085592354674942,11
+FRA,1473,1946-1958,1946,1958,PRL,Republican Liberty Party,0.8756,0.013727516377056447,17
+FRA,1580,1946-1958,1946,1958,POUJ,Poujadists,0.995,0.01,22
+ISL,1249,1919-1939,1919,1939,COMM,Communist People Alliance,0.005,0.01,6
+ISL,1325,1919-1939,1919,1939,SOCd,Social Democratic Party,0.2101,0.017350552344714183,6
+ISL,964,1919-1939,1919,1939,PROG,Progressives/Farmers Party,0.6211,0.04992876592373045,6
+ISL,1890,1919-1939,1919,1939,LIB,Liberal Party,0.6302,0.0441316401991436,6
+ISL,363,1919-1939,1919,1939,INDP,Independence Party,0.8128,0.02135138559126004,6
+ISL,1891,1919-1939,1919,1939,CONS,Conservative Party,0.995,0.01,6
+ISL,1249,1945-1973,1945,1973,COMM,Communist People Alliance,0.005,0.01,9
+ISL,1517,1945-1973,1945,1973,LLIB,Liberal Left,0.11410000000000001,0.012966666666666668,9
+ISL,1325,1945-1973,1945,1973,SOCd,Social Democratic Party,0.3778,0.029333333333333336,9
+ISL,964,1945-1973,1945,1973,PROG,Progressive Party,0.7299,0.025666666666666667,9
+ISL,363,1945-1973,1945,1973,INDP,Independence Party,0.995,0.01,9
+ISR,907,1949-1973,1949,1973,RAKA,Rakah,0.005,0.01,10
+ISR,1417,1949-1973,1949,1973,MAKI,Maki,0.0346,0.01,10
+ISR,169,1949-1973,1949,1973,MAPM,Mapam,0.1804,0.018499324311985015,10
+ISR,1398,1949-1973,1949,1973,ADUT,Ahdut Haavoda,0.3492,0.03993956684792663,10
+ISR,615,1949-1973,1949,1973,LAB,Labor Alignment,0.4808,0.021029146440119724,10
+ISR,109,1949-1973,1949,1973,MAPI,Mapai,0.4827,0.02210432084457697,10
+ISR,1491,1949-1973,1949,1973,PAUG,Poalei Agudat,0.5582,0.021392210420306423,6
+ISR,396,1949-1973,1949,1973,RAFI,Rafi,0.6042000000000001,0.025361416138919145,7
+ISR,712,1949-1973,1949,1973,PROG,Progressives,0.65,0.01902117241391813,8
+ISR,434,1949-1973,1949,1973,ILIB,Independent Liberals,0.6579999999999999,0.025166666666666667,9
+ISR,1447,1949-1973,1949,1973,NRP,National Religious Party,0.69,0.031030883234057557,7
+ISR,909,1949-1973,1949,1973,URF,United Religious Front,0.77,0.04753880520164553,5
+ISR,678,1949-1973,1949,1973,LIB,Liberals,0.7857,0.034499999999999996,9
+ISR,770,1949-1973,1949,1973,NATL,National List,0.7967,0.04437658917342191,6
+ISR,1786,1949-1973,1949,1973,TORA,Torah,0.8529000000000001,0.01,6
+ISR,1489,1949-1973,1949,1973,ZION,General Zionists,0.8795000000000001,0.011166666666666667,9
+ISR,198,1949-1973,1949,1973,GHAL,Gahal,0.9068999999999999,0.016652364696943193,8
+ISR,641,1949-1973,1949,1973,AGDT,Agudat Israel,0.9625,0.012298373876248842,5
+ISR,1655,1949-1973,1949,1973,HRUT,Herut,0.98,0.01,9
+ITA,34,1946-1975,1946,1975,PCI,Communist Party,0.0308,0.01,12
+ITA,1505,1946-1975,1946,1975,PSIU,Socialist Party of Proletarian Unity,0.0361,0.011547005383792516,12
+ITA,742,1946-1975,1946,1975,PSI,Socialist Party (Nenni),0.2281,0.01,12
+ITA,1126,1946-1975,1946,1975,PSDI,Social Democratic Party,0.36119999999999997,0.010883052574224445,12
+ITA,394,1946-1975,1946,1975,PRI,Republican Party,0.4218,0.01,12
+ITA,934,1946-1975,1946,1975,DC,Christian Democratic Party,0.5716,0.015703927321957824,12
+ITA,1461,1946-1975,1946,1975,PLI,Liberal Party,0.7292000000000001,0.01,12
+ITA,773,1946-1975,1946,1975,MON,Monarchists,0.9168000000000001,0.01,12
+ITA,1696,1946-1975,1946,1975,MSI,Italian Social Movement,0.995,0.01,12
+LUX,1647,1945-1973,1945,1973,COMM,Communist Party,0.005,0.01,7
+LUX,186,1945-1973,1945,1973,SOCd,Socialist Party,0.36219999999999997,0.04849284188708385,7
+LUX,539,1945-1973,1945,1973,CSOC,Christian Social Party,0.8576,0.05091181451434291,7
+LUX,300,1945-1973,1945,1973,GRPD,Democratic Group,0.9333,0.051653170280245145,5
+NLD,459,1919-1939,1919,1939,CPN,Communist Party,0.005,0.01,6
+NLD,1894,1919-1939,1919,1939,SOCd,Social Democratic Party (SDAP),0.1832,0.022086232514094993,6
+NLD,1581,1919-1939,1919,1939,RAD,Radical Party,0.4704,0.012818996320565301,6
+NLD,1390,1919-1939,1919,1939,KVP,Catholic Party,0.5605,0.02629118990587278,6
+NLD,1102,1919-1939,1919,1939,CHU,Christian Historical Union,0.6920000000000001,0.030986045246207204,6
+NLD,1889,1919-1939,1919,1939,LIB,Liberal Party,0.7193999999999999,0.016738179909018384,6
+NLD,163,1919-1939,1919,1939,ARP,Anti-Revolutionary Party,0.7243,0.03715059443221154,6
+NLD,1178,1919-1939,1919,1939,SGP,Political Reformed Party,0.885,0.01,6
+NLD,1832,1919-1939,1919,1939,NSB,National Social Movement,0.995,0.01,6
+NLD,459,1945-1973,1945,1973,CPN,Communist Party,0.005,0.01,14
+NLD,1050,1945-1973,1945,1973,PSP,Pacifist Socialist Party,0.0766,0.012427647748927736,14
+NLD,1581,1945-1973,1945,1973,PPR,Political Radicals,0.1967,0.01501526495997263,11
+NLD,1234,1945-1973,1945,1973,PVDA,Labor Party,0.2669,0.01312252697790004,14
+NLD,921,1945-1973,1945,1973,DS70,Social Democratic splinter,0.4604,0.016436566377614103,14
+NLD,1390,1945-1973,1945,1973,KVP,Catholic Party,0.5125,0.01,14
+NLD,163,1945-1973,1945,1973,ARP,Anti-Revolutionary Party,0.5556,0.022931014556086014,14
+NLD,1102,1945-1973,1945,1973,CHU,Christian Historical Union,0.6647,0.015100260168051978,14
+NLD,828,1945-1973,1945,1973,VVD,Liberal Party,0.7609999999999999,0.0155546042793031,14
+NLD,1178,1945-1973,1945,1973,SGP,Political Reform Party,0.9634999999999999,0.010650243767524398,13
+NLD,1602,1945-1973,1945,1973,GPV,Reformed Political Association,0.9670000000000001,0.011200595222278741,12
+NLD,1110,1945-1973,1945,1973,BP,Peasant Party (Poujadist),0.9682,0.013447405968168258,11
+NOR,448,1919-1939,1919,1939,LAB,Labor Party,0.005,0.01,19
+NOR,1173,1919-1939,1919,1939,LIB,Liberal Party,0.5771000000000001,0.017137355320130963,19
+NOR,1072,1919-1939,1919,1939,AGR,Agrarian Party,0.7061,0.019821519406416535,19
+NOR,503,1919-1939,1919,1939,CONS,Conservative Party,0.995,0.01,19
+NOR,1079,1945-1973,1945,1973,COMM,Communist Party,0.005,0.01,20
+NOR,1203,1945-1973,1945,1973,SOCL,Socialist People Party,0.0745,0.01,20
+NOR,448,1945-1973,1945,1973,LAB,Labor Party,0.3529,0.011292143286373936,20
+NOR,1173,1945-1973,1945,1973,LIB,Liberal Party,0.6426999999999999,0.015294704966098562,20
+NOR,705,1945-1973,1945,1973,CHPP,Christian People Party,0.6907,0.020280350874157655,19
+NOR,1072,1945-1973,1945,1973,CENT,Center Party,0.7813,0.01,20
+NOR,503,1945-1973,1945,1973,CONS,Conservative Party,0.995,0.01,20
+SWE,830,1919-1939,1919,1939,COMM,Communist Party,0.005,0.01,15
+SWE,487,1919-1939,1919,1939,SOCd,Social Democratic Party,0.2585,0.013891100268397268,15
+SWE,199,1919-1939,1919,1939,AGR,Agrarian Party,0.5857,0.014020199713270847,15
+SWE,1274,1919-1939,1919,1939,LIB,Liberal Party,0.7419,0.011386571037849805,15
+SWE,690,1919-1939,1919,1939,CONS,Conservative Party,0.995,0.01,15
+SWE,830,1945-1973,1945,1973,COMM,Communist Party,0.005,0.01,20
+SWE,487,1945-1973,1945,1973,SOCd,Social Democratic Party,0.3251,0.013796539421173702,20
+SWE,199,1945-1973,1945,1973,CENT,Center/Agrarian Party,0.6456999999999999,0.01,19
+SWE,1274,1945-1973,1945,1973,LIB,Liberal/People Party,0.7444,0.013863621460498696,20
+SWE,690,1945-1973,1945,1973,CONS,Conservative Party,0.995,0.01,20

data/morgan_lr.csv

@@ -0,0 +1,472 @@
+country,party,var,year,val,project,n_scale,val_int,n_experts
+BE,49,lr_morgan,1971,0.995,Morgan,10,10,17
+BE,281,lr_morgan,1968,0.5365,Morgan,10,5,16
+BE,281,lr_morgan,1971,0.5365,Morgan,10,5,16
+BE,405,lr_morgan,1946,0.7090000000000001,Morgan,10,7,17
+BE,405,lr_morgan,1949,0.7090000000000001,Morgan,10,7,17
+BE,405,lr_morgan,1950,0.7090000000000001,Morgan,10,7,17
+BE,405,lr_morgan,1954,0.7090000000000001,Morgan,10,7,17
+BE,405,lr_morgan,1958,0.7090000000000001,Morgan,10,7,17
+BE,405,lr_morgan,1961,0.7090000000000001,Morgan,10,7,17
+BE,405,lr_morgan,1965,0.7090000000000001,Morgan,10,7,17
+BE,480,lr_morgan,1946,0.3193,Morgan,10,3,17
+BE,480,lr_morgan,1949,0.3193,Morgan,10,3,17
+BE,480,lr_morgan,1950,0.3193,Morgan,10,3,17
+BE,480,lr_morgan,1954,0.3193,Morgan,10,3,17
+BE,480,lr_morgan,1958,0.3193,Morgan,10,3,17
+BE,480,lr_morgan,1961,0.3193,Morgan,10,3,17
+BE,480,lr_morgan,1965,0.3193,Morgan,10,3,17
+BE,480,lr_morgan,1968,0.3193,Morgan,10,3,17
+BE,480,lr_morgan,1971,0.3193,Morgan,10,3,17
+BE,1424,lr_morgan,1954,0.8321,Morgan,10,8,17
+BE,1424,lr_morgan,1958,0.8321,Morgan,10,8,17
+BE,1424,lr_morgan,1961,0.8321,Morgan,10,8,17
+BE,1424,lr_morgan,1965,0.8321,Morgan,10,8,17
+BE,1424,lr_morgan,1968,0.8321,Morgan,10,8,17
+BE,1424,lr_morgan,1971,0.8321,Morgan,10,8,17
+DK,277,lr_morgan,1945,0.005,Morgan,10,0,22
+DK,277,lr_morgan,1947,0.005,Morgan,10,0,22
+DK,277,lr_morgan,1950,0.005,Morgan,10,0,22
+DK,277,lr_morgan,1953,0.005,Morgan,10,0,22
+DK,277,lr_morgan,1957,0.005,Morgan,10,0,22
+DK,277,lr_morgan,1960,0.005,Morgan,10,0,22
+DK,277,lr_morgan,1964,0.005,Morgan,10,0,22
+DK,277,lr_morgan,1966,0.005,Morgan,10,0,22
+DK,277,lr_morgan,1968,0.005,Morgan,10,0,22
+DK,277,lr_morgan,1971,0.005,Morgan,10,0,22
+DK,277,lr_morgan,1973,0.005,Morgan,10,0,22
+DK,329,lr_morgan,1960,0.1954,Morgan,10,2,22
+DK,329,lr_morgan,1964,0.1954,Morgan,10,2,22
+DK,329,lr_morgan,1966,0.1954,Morgan,10,2,22
+DK,329,lr_morgan,1968,0.1954,Morgan,10,2,22
+DK,329,lr_morgan,1971,0.1954,Morgan,10,2,22
+DK,329,lr_morgan,1973,0.1954,Morgan,10,2,22
+DK,379,lr_morgan,1945,0.41490000000000005,Morgan,10,4,22
+DK,379,lr_morgan,1947,0.41490000000000005,Morgan,10,4,22
+DK,379,lr_morgan,1950,0.41490000000000005,Morgan,10,4,22
+DK,379,lr_morgan,1953,0.41490000000000005,Morgan,10,4,22
+DK,379,lr_morgan,1957,0.41490000000000005,Morgan,10,4,22
+DK,379,lr_morgan,1960,0.41490000000000005,Morgan,10,4,22
+DK,379,lr_morgan,1964,0.41490000000000005,Morgan,10,4,22
+DK,379,lr_morgan,1966,0.41490000000000005,Morgan,10,4,22
+DK,379,lr_morgan,1968,0.41490000000000005,Morgan,10,4,22
+DK,379,lr_morgan,1971,0.41490000000000005,Morgan,10,4,22
+DK,379,lr_morgan,1973,0.41490000000000005,Morgan,10,4,22
+DK,536,lr_morgan,1945,0.995,Morgan,10,10,22
+DK,536,lr_morgan,1947,0.995,Morgan,10,10,22
+DK,536,lr_morgan,1950,0.995,Morgan,10,10,22
+DK,536,lr_morgan,1953,0.995,Morgan,10,10,22
+DK,536,lr_morgan,1957,0.995,Morgan,10,10,22
+DK,536,lr_morgan,1960,0.995,Morgan,10,10,22
+DK,536,lr_morgan,1964,0.995,Morgan,10,10,22
+DK,536,lr_morgan,1966,0.995,Morgan,10,10,22
+DK,536,lr_morgan,1968,0.995,Morgan,10,10,22
+DK,536,lr_morgan,1971,0.995,Morgan,10,10,22
+DK,536,lr_morgan,1973,0.995,Morgan,10,10,22
+DK,1134,lr_morgan,1945,0.8408,Morgan,10,8,17
+DK,1134,lr_morgan,1947,0.8408,Morgan,10,8,17
+DK,1134,lr_morgan,1950,0.8408,Morgan,10,8,17
+DK,1134,lr_morgan,1953,0.8408,Morgan,10,8,17
+DK,1134,lr_morgan,1957,0.8408,Morgan,10,8,17
+DK,1134,lr_morgan,1960,0.8408,Morgan,10,8,17
+DK,1134,lr_morgan,1964,0.8408,Morgan,10,8,17
+DK,1134,lr_morgan,1966,0.8408,Morgan,10,8,17
+DK,1134,lr_morgan,1968,0.8408,Morgan,10,8,17
+DK,1134,lr_morgan,1971,0.8408,Morgan,10,8,17
+DK,1134,lr_morgan,1973,0.8408,Morgan,10,8,17
+DK,1136,lr_morgan,1968,0.065,Morgan,10,1,22
+DK,1136,lr_morgan,1971,0.065,Morgan,10,1,22
+DK,1136,lr_morgan,1973,0.065,Morgan,10,1,22
+DK,1204,lr_morgan,1945,0.8198000000000001,Morgan,10,8,22
+DK,1204,lr_morgan,1947,0.8198000000000001,Morgan,10,8,22
+DK,1204,lr_morgan,1950,0.8198000000000001,Morgan,10,8,22
+DK,1204,lr_morgan,1953,0.8198000000000001,Morgan,10,8,22
+DK,1204,lr_morgan,1957,0.8198000000000001,Morgan,10,8,22
+DK,1204,lr_morgan,1960,0.8198000000000001,Morgan,10,8,22
+DK,1204,lr_morgan,1964,0.8198000000000001,Morgan,10,8,22
+DK,1204,lr_morgan,1966,0.8198000000000001,Morgan,10,8,22
+DK,1204,lr_morgan,1968,0.8198000000000001,Morgan,10,8,22
+DK,1204,lr_morgan,1971,0.8198000000000001,Morgan,10,8,22
+DK,1204,lr_morgan,1973,0.8198000000000001,Morgan,10,8,22
+DK,1507,lr_morgan,1945,0.6185,Morgan,10,6,22
+DK,1507,lr_morgan,1947,0.6185,Morgan,10,6,22
+DK,1507,lr_morgan,1950,0.6185,Morgan,10,6,22
+DK,1507,lr_morgan,1953,0.6185,Morgan,10,6,22
+DK,1507,lr_morgan,1957,0.6185,Morgan,10,6,22
+DK,1507,lr_morgan,1960,0.6185,Morgan,10,6,22
+DK,1507,lr_morgan,1964,0.6185,Morgan,10,6,22
+DK,1507,lr_morgan,1966,0.6185,Morgan,10,6,22
+DK,1507,lr_morgan,1968,0.6185,Morgan,10,6,22
+DK,1507,lr_morgan,1971,0.6185,Morgan,10,6,22
+DK,1507,lr_morgan,1973,0.6185,Morgan,10,6,22
+FI,249,lr_morgan,1945,0.7034,Morgan,10,7,15
+FI,249,lr_morgan,1948,0.7034,Morgan,10,7,15
+FI,249,lr_morgan,1951,0.7034,Morgan,10,7,15
+FI,249,lr_morgan,1954,0.7034,Morgan,10,7,15
+FI,249,lr_morgan,1958,0.7034,Morgan,10,7,15
+FI,249,lr_morgan,1962,0.7034,Morgan,10,7,15
+FI,249,lr_morgan,1966,0.7034,Morgan,10,7,15
+FI,249,lr_morgan,1970,0.7034,Morgan,10,7,15
+FI,249,lr_morgan,1972,0.7034,Morgan,10,7,15
+FI,495,lr_morgan,1945,0.9854999999999999,Morgan,10,10,15
+FI,495,lr_morgan,1948,0.9854999999999999,Morgan,10,10,15
+FI,495,lr_morgan,1951,0.9854999999999999,Morgan,10,10,15
+FI,495,lr_morgan,1954,0.9854999999999999,Morgan,10,10,15
+FI,495,lr_morgan,1958,0.9854999999999999,Morgan,10,10,15
+FI,495,lr_morgan,1962,0.9854999999999999,Morgan,10,10,15
+FI,495,lr_morgan,1966,0.9854999999999999,Morgan,10,10,15
+FI,495,lr_morgan,1970,0.9854999999999999,Morgan,10,10,15
+FI,495,lr_morgan,1972,0.9854999999999999,Morgan,10,10,15
+FI,901,lr_morgan,1945,0.5776,Morgan,10,6,13
+FI,901,lr_morgan,1948,0.5776,Morgan,10,6,13
+FI,901,lr_morgan,1951,0.5776,Morgan,10,6,13
+FI,901,lr_morgan,1954,0.5776,Morgan,10,6,13
+FI,901,lr_morgan,1958,0.5776,Morgan,10,6,13
+FI,901,lr_morgan,1962,0.5776,Morgan,10,6,13
+FI,901,lr_morgan,1966,0.5776,Morgan,10,6,13
+FI,901,lr_morgan,1970,0.5776,Morgan,10,6,13
+FI,901,lr_morgan,1972,0.5776,Morgan,10,6,13
+FI,1096,lr_morgan,1945,0.005,Morgan,10,0,15
+FI,1096,lr_morgan,1948,0.005,Morgan,10,0,15
+FI,1096,lr_morgan,1951,0.005,Morgan,10,0,15
+FI,1096,lr_morgan,1954,0.005,Morgan,10,0,15
+FI,1096,lr_morgan,1958,0.005,Morgan,10,0,15
+FI,1096,lr_morgan,1962,0.005,Morgan,10,0,15
+FI,1096,lr_morgan,1966,0.005,Morgan,10,0,15
+FI,1096,lr_morgan,1970,0.005,Morgan,10,0,15
+FI,1096,lr_morgan,1972,0.005,Morgan,10,0,15
+FI,1164,lr_morgan,1958,0.2115,Morgan,10,2,15
+FI,1164,lr_morgan,1962,0.2115,Morgan,10,2,15
+FI,1164,lr_morgan,1966,0.2115,Morgan,10,2,15
+FI,1229,lr_morgan,1945,0.7975,Morgan,10,8,15
+FI,1229,lr_morgan,1948,0.7975,Morgan,10,8,15
+FI,1229,lr_morgan,1951,0.7975,Morgan,10,8,15
+FI,1229,lr_morgan,1954,0.7975,Morgan,10,8,15
+FI,1229,lr_morgan,1958,0.7975,Morgan,10,8,15
+FI,1229,lr_morgan,1962,0.7975,Morgan,10,8,15
+FI,1229,lr_morgan,1966,0.7975,Morgan,10,8,15
+FI,1229,lr_morgan,1970,0.7975,Morgan,10,8,15
+FI,1229,lr_morgan,1972,0.7975,Morgan,10,8,15
+FI,1303,lr_morgan,1945,0.25780000000000003,Morgan,10,3,15
+FI,1303,lr_morgan,1948,0.25780000000000003,Morgan,10,3,15
+FI,1303,lr_morgan,1951,0.25780000000000003,Morgan,10,3,15
+FI,1303,lr_morgan,1954,0.25780000000000003,Morgan,10,3,15
+FI,1303,lr_morgan,1958,0.25780000000000003,Morgan,10,3,15
+FI,1303,lr_morgan,1962,0.25780000000000003,Morgan,10,3,15
+FI,1303,lr_morgan,1966,0.25780000000000003,Morgan,10,3,15
+FI,1303,lr_morgan,1970,0.25780000000000003,Morgan,10,3,15
+FI,1303,lr_morgan,1972,0.25780000000000003,Morgan,10,3,15
+FI,1689,lr_morgan,1966,0.6411,Morgan,10,6,13
+FI,1689,lr_morgan,1970,0.6411,Morgan,10,6,13
+FI,1689,lr_morgan,1972,0.6411,Morgan,10,6,13
+FR,14,lr_morgan,1946,0.4337,Morgan,10,4,20
+FR,14,lr_morgan,1951,0.4337,Morgan,10,4,20
+FR,14,lr_morgan,1956,0.4337,Morgan,10,4,20
+FR,14,lr_morgan,1958,0.4337,Morgan,10,4,20
+FR,737,lr_morgan,1946,0.43119999999999997,Morgan,10,4,23
+FR,737,lr_morgan,1951,0.43119999999999997,Morgan,10,4,23
+FR,737,lr_morgan,1956,0.43119999999999997,Morgan,10,4,23
+FR,737,lr_morgan,1958,0.43119999999999997,Morgan,10,4,23
+FR,1083,lr_morgan,1946,0.7,Morgan,10,7,23
+FR,1083,lr_morgan,1951,0.7,Morgan,10,7,23
+FR,1083,lr_morgan,1956,0.7,Morgan,10,7,23
+FR,1083,lr_morgan,1958,0.7,Morgan,10,7,23
+FR,1251,lr_morgan,1946,0.005,Morgan,10,0,23
+FR,1251,lr_morgan,1951,0.005,Morgan,10,0,23
+FR,1251,lr_morgan,1956,0.005,Morgan,10,0,23
+FR,1251,lr_morgan,1958,0.005,Morgan,10,0,23
+FR,1478,lr_morgan,1946,0.2365,Morgan,10,2,23
+FR,1478,lr_morgan,1951,0.2365,Morgan,10,2,23
+FR,1478,lr_morgan,1956,0.2365,Morgan,10,2,23
+FR,1478,lr_morgan,1958,0.2365,Morgan,10,2,23
+FR,1629,lr_morgan,1946,0.835,Morgan,10,8,19
+FR,1629,lr_morgan,1951,0.835,Morgan,10,8,19
+FR,1629,lr_morgan,1956,0.835,Morgan,10,8,19
+FR,1629,lr_morgan,1958,0.835,Morgan,10,8,19
+IL,109,lr_morgan,1949,0.4827,Morgan,10,5,10
+IL,109,lr_morgan,1951,0.4827,Morgan,10,5,10
+IL,109,lr_morgan,1955,0.4827,Morgan,10,5,10
+IL,109,lr_morgan,1959,0.4827,Morgan,10,5,10
+IL,109,lr_morgan,1961,0.4827,Morgan,10,5,10
+IL,169,lr_morgan,1949,0.1804,Morgan,10,2,10
+IL,169,lr_morgan,1951,0.1804,Morgan,10,2,10
+IL,169,lr_morgan,1955,0.1804,Morgan,10,2,10
+IL,169,lr_morgan,1959,0.1804,Morgan,10,2,10
+IL,169,lr_morgan,1961,0.1804,Morgan,10,2,10
+IL,169,lr_morgan,1965,0.1804,Morgan,10,2,10
+IL,434,lr_morgan,1965,0.6579999999999999,Morgan,10,7,9
+IL,434,lr_morgan,1969,0.6579999999999999,Morgan,10,7,9
+IL,434,lr_morgan,1973,0.6579999999999999,Morgan,10,7,9
+IL,615,lr_morgan,1965,0.4808,Morgan,10,5,10
+IL,615,lr_morgan,1969,0.4808,Morgan,10,5,10
+IL,615,lr_morgan,1973,0.4808,Morgan,10,5,10
+IL,641,lr_morgan,1949,0.9625,Morgan,10,10,5
+IL,641,lr_morgan,1951,0.9625,Morgan,10,10,5
+IL,641,lr_morgan,1955,0.9625,Morgan,10,10,5
+IL,641,lr_morgan,1959,0.9625,Morgan,10,10,5
+IL,641,lr_morgan,1965,0.9625,Morgan,10,10,5
+IL,641,lr_morgan,1969,0.9625,Morgan,10,10,5
+IL,907,lr_morgan,1965,0.005,Morgan,10,0,10
+IL,907,lr_morgan,1969,0.005,Morgan,10,0,10
+IL,907,lr_morgan,1973,0.005,Morgan,10,0,10
+IL,1398,lr_morgan,1955,0.3492,Morgan,10,3,10
+IL,1398,lr_morgan,1959,0.3492,Morgan,10,3,10
+IL,1398,lr_morgan,1961,0.3492,Morgan,10,3,10
+IL,1417,lr_morgan,1949,0.0346,Morgan,10,0,10
+IL,1417,lr_morgan,1951,0.0346,Morgan,10,0,10
+IL,1417,lr_morgan,1955,0.0346,Morgan,10,0,10
+IL,1417,lr_morgan,1959,0.0346,Morgan,10,0,10
+IL,1417,lr_morgan,1961,0.0346,Morgan,10,0,10
+IL,1417,lr_morgan,1965,0.0346,Morgan,10,0,10
+IL,1417,lr_morgan,1969,0.0346,Morgan,10,0,10
+IL,1447,lr_morgan,1955,0.69,Morgan,10,7,7
+IL,1447,lr_morgan,1959,0.69,Morgan,10,7,7
+IL,1447,lr_morgan,1961,0.69,Morgan,10,7,7
+IL,1447,lr_morgan,1965,0.69,Morgan,10,7,7
+IL,1447,lr_morgan,1969,0.69,Morgan,10,7,7
+IL,1447,lr_morgan,1973,0.69,Morgan,10,7,7
+IL,1489,lr_morgan,1949,0.8795000000000001,Morgan,10,9,9
+IL,1489,lr_morgan,1951,0.8795000000000001,Morgan,10,9,9
+IL,1489,lr_morgan,1955,0.8795000000000001,Morgan,10,9,9
+IL,1489,lr_morgan,1959,0.8795000000000001,Morgan,10,9,9
+IL,1655,lr_morgan,1949,0.98,Morgan,10,10,9
+IL,1655,lr_morgan,1951,0.98,Morgan,10,10,9
+IL,1655,lr_morgan,1955,0.98,Morgan,10,10,9
+IL,1655,lr_morgan,1959,0.98,Morgan,10,10,9
+IL,1655,lr_morgan,1961,0.98,Morgan,10,10,9
+IS,363,lr_morgan,1946,0.995,Morgan,10,10,9
+IS,363,lr_morgan,1949,0.995,Morgan,10,10,9
+IS,363,lr_morgan,1953,0.995,Morgan,10,10,9
+IS,363,lr_morgan,1956,0.995,Morgan,10,10,9
+IS,363,lr_morgan,1959,0.995,Morgan,10,10,9
+IS,363,lr_morgan,1963,0.995,Morgan,10,10,9
+IS,363,lr_morgan,1967,0.995,Morgan,10,10,9
+IS,363,lr_morgan,1971,0.995,Morgan,10,10,9
+IS,964,lr_morgan,1946,0.7299,Morgan,10,7,9
+IS,964,lr_morgan,1949,0.7299,Morgan,10,7,9
+IS,964,lr_morgan,1953,0.7299,Morgan,10,7,9
+IS,964,lr_morgan,1956,0.7299,Morgan,10,7,9
+IS,964,lr_morgan,1959,0.7299,Morgan,10,7,9
+IS,964,lr_morgan,1963,0.7299,Morgan,10,7,9
+IS,964,lr_morgan,1967,0.7299,Morgan,10,7,9
+IS,964,lr_morgan,1971,0.7299,Morgan,10,7,9
+IS,1249,lr_morgan,1946,0.005,Morgan,10,0,9
+IS,1249,lr_morgan,1949,0.005,Morgan,10,0,9
+IS,1249,lr_morgan,1953,0.005,Morgan,10,0,9
+IS,1249,lr_morgan,1956,0.005,Morgan,10,0,9
+IS,1249,lr_morgan,1959,0.005,Morgan,10,0,9
+IS,1249,lr_morgan,1963,0.005,Morgan,10,0,9
+IS,1249,lr_morgan,1967,0.005,Morgan,10,0,9
+IS,1249,lr_morgan,1971,0.005,Morgan,10,0,9
+IS,1325,lr_morgan,1946,0.3778,Morgan,10,4,9
+IS,1325,lr_morgan,1949,0.3778,Morgan,10,4,9
+IS,1325,lr_morgan,1953,0.3778,Morgan,10,4,9
+IS,1325,lr_morgan,1956,0.3778,Morgan,10,4,9
+IS,1325,lr_morgan,1959,0.3778,Morgan,10,4,9
+IS,1325,lr_morgan,1963,0.3778,Morgan,10,4,9
+IS,1325,lr_morgan,1967,0.3778,Morgan,10,4,9
+IS,1325,lr_morgan,1971,0.3778,Morgan,10,4,9
+IS,1517,lr_morgan,1967,0.11410000000000001,Morgan,10,1,9
+IS,1517,lr_morgan,1971,0.11410000000000001,Morgan,10,1,9
+IT,394,lr_morgan,1946,0.4218,Morgan,10,4,12
+IT,394,lr_morgan,1948,0.4218,Morgan,10,4,12
+IT,394,lr_morgan,1953,0.4218,Morgan,10,4,12
+IT,394,lr_morgan,1958,0.4218,Morgan,10,4,12
+IT,394,lr_morgan,1963,0.4218,Morgan,10,4,12
+IT,394,lr_morgan,1968,0.4218,Morgan,10,4,12
+IT,394,lr_morgan,1972,0.4218,Morgan,10,4,12
+IT,742,lr_morgan,1946,0.2281,Morgan,10,2,12
+IT,742,lr_morgan,1948,0.2281,Morgan,10,2,12
+IT,742,lr_morgan,1953,0.2281,Morgan,10,2,12
+IT,742,lr_morgan,1958,0.2281,Morgan,10,2,12
+IT,742,lr_morgan,1963,0.2281,Morgan,10,2,12
+IT,742,lr_morgan,1972,0.2281,Morgan,10,2,12
+IT,934,lr_morgan,1946,0.5716,Morgan,10,6,12
+IT,934,lr_morgan,1948,0.5716,Morgan,10,6,12
+IT,934,lr_morgan,1953,0.5716,Morgan,10,6,12
+IT,934,lr_morgan,1958,0.5716,Morgan,10,6,12
+IT,934,lr_morgan,1963,0.5716,Morgan,10,6,12
+IT,934,lr_morgan,1968,0.5716,Morgan,10,6,12
+IT,934,lr_morgan,1972,0.5716,Morgan,10,6,12
+IT,1126,lr_morgan,1948,0.36119999999999997,Morgan,10,4,12
+IT,1126,lr_morgan,1953,0.36119999999999997,Morgan,10,4,12
+IT,1126,lr_morgan,1958,0.36119999999999997,Morgan,10,4,12
+IT,1126,lr_morgan,1963,0.36119999999999997,Morgan,10,4,12
+IT,1126,lr_morgan,1972,0.36119999999999997,Morgan,10,4,12
+IT,1461,lr_morgan,1946,0.7292000000000001,Morgan,10,7,12
+IT,1461,lr_morgan,1948,0.7292000000000001,Morgan,10,7,12
+IT,1461,lr_morgan,1953,0.7292000000000001,Morgan,10,7,12
+IT,1461,lr_morgan,1958,0.7292000000000001,Morgan,10,7,12
+IT,1461,lr_morgan,1963,0.7292000000000001,Morgan,10,7,12
+IT,1461,lr_morgan,1968,0.7292000000000001,Morgan,10,7,12
+IT,1461,lr_morgan,1972,0.7292000000000001,Morgan,10,7,12
+LU,186,lr_morgan,1945,0.36219999999999997,Morgan,10,4,7
+LU,186,lr_morgan,1948,0.36219999999999997,Morgan,10,4,7
+LU,186,lr_morgan,1951,0.36219999999999997,Morgan,10,4,7
+LU,186,lr_morgan,1954,0.36219999999999997,Morgan,10,4,7
+LU,186,lr_morgan,1959,0.36219999999999997,Morgan,10,4,7
+LU,186,lr_morgan,1964,0.36219999999999997,Morgan,10,4,7
+LU,186,lr_morgan,1968,0.36219999999999997,Morgan,10,4,7
+LU,300,lr_morgan,1945,0.9333,Morgan,10,9,5
+LU,300,lr_morgan,1948,0.9333,Morgan,10,9,5
+LU,300,lr_morgan,1951,0.9333,Morgan,10,9,5
+LU,300,lr_morgan,1954,0.9333,Morgan,10,9,5
+LU,300,lr_morgan,1959,0.9333,Morgan,10,9,5
+LU,300,lr_morgan,1964,0.9333,Morgan,10,9,5
+LU,300,lr_morgan,1968,0.9333,Morgan,10,9,5
+LU,539,lr_morgan,1945,0.8576,Morgan,10,9,7
+LU,539,lr_morgan,1948,0.8576,Morgan,10,9,7
+LU,539,lr_morgan,1951,0.8576,Morgan,10,9,7
+LU,539,lr_morgan,1954,0.8576,Morgan,10,9,7
+LU,539,lr_morgan,1959,0.8576,Morgan,10,9,7
+LU,539,lr_morgan,1964,0.8576,Morgan,10,9,7
+LU,539,lr_morgan,1968,0.8576,Morgan,10,9,7
+LU,1647,lr_morgan,1945,0.005,Morgan,10,0,7
+LU,1647,lr_morgan,1948,0.005,Morgan,10,0,7
+LU,1647,lr_morgan,1951,0.005,Morgan,10,0,7
+LU,1647,lr_morgan,1954,0.005,Morgan,10,0,7
+LU,1647,lr_morgan,1959,0.005,Morgan,10,0,7
+LU,1647,lr_morgan,1964,0.005,Morgan,10,0,7
+LU,1647,lr_morgan,1968,0.005,Morgan,10,0,7
+NL,163,lr_morgan,1946,0.5556,Morgan,10,6,14
+NL,163,lr_morgan,1948,0.5556,Morgan,10,6,14
+NL,163,lr_morgan,1952,0.5556,Morgan,10,6,14
+NL,163,lr_morgan,1956,0.5556,Morgan,10,6,14
+NL,163,lr_morgan,1959,0.5556,Morgan,10,6,14
+NL,163,lr_morgan,1963,0.5556,Morgan,10,6,14
+NL,163,lr_morgan,1967,0.5556,Morgan,10,6,14
+NL,163,lr_morgan,1971,0.5556,Morgan,10,6,14
+NL,163,lr_morgan,1972,0.5556,Morgan,10,6,14
+NL,459,lr_morgan,1946,0.005,Morgan,10,0,14
+NL,459,lr_morgan,1948,0.005,Morgan,10,0,14
+NL,828,lr_morgan,1946,0.7609999999999999,Morgan,10,8,14
+NL,828,lr_morgan,1948,0.7609999999999999,Morgan,10,8,14
+NL,828,lr_morgan,1952,0.7609999999999999,Morgan,10,8,14
+NL,828,lr_morgan,1956,0.7609999999999999,Morgan,10,8,14
+NL,828,lr_morgan,1959,0.7609999999999999,Morgan,10,8,14
+NL,828,lr_morgan,1963,0.7609999999999999,Morgan,10,8,14
+NL,828,lr_morgan,1967,0.7609999999999999,Morgan,10,8,14
+NL,828,lr_morgan,1971,0.7609999999999999,Morgan,10,8,14
+NL,828,lr_morgan,1972,0.7609999999999999,Morgan,10,8,14
+NL,921,lr_morgan,1971,0.4604,Morgan,10,5,14
+NL,921,lr_morgan,1972,0.4604,Morgan,10,5,14
+NL,1102,lr_morgan,1946,0.6647,Morgan,10,7,14
+NL,1102,lr_morgan,1948,0.6647,Morgan,10,7,14
+NL,1102,lr_morgan,1952,0.6647,Morgan,10,7,14
+NL,1102,lr_morgan,1956,0.6647,Morgan,10,7,14
+NL,1102,lr_morgan,1959,0.6647,Morgan,10,7,14
+NL,1102,lr_morgan,1963,0.6647,Morgan,10,7,14
+NL,1102,lr_morgan,1967,0.6647,Morgan,10,7,14
+NL,1102,lr_morgan,1971,0.6647,Morgan,10,7,14
+NL,1102,lr_morgan,1972,0.6647,Morgan,10,7,14
+NL,1178,lr_morgan,1946,0.9634999999999999,Morgan,10,10,13
+NL,1178,lr_morgan,1948,0.9634999999999999,Morgan,10,10,13
+NL,1234,lr_morgan,1946,0.2669,Morgan,10,3,14
+NL,1234,lr_morgan,1948,0.2669,Morgan,10,3,14
+NL,1234,lr_morgan,1952,0.2669,Morgan,10,3,14
+NL,1234,lr_morgan,1956,0.2669,Morgan,10,3,14
+NL,1234,lr_morgan,1959,0.2669,Morgan,10,3,14
+NL,1234,lr_morgan,1963,0.2669,Morgan,10,3,14
+NL,1234,lr_morgan,1967,0.2669,Morgan,10,3,14
+NL,1234,lr_morgan,1971,0.2669,Morgan,10,3,14
+NL,1234,lr_morgan,1972,0.2669,Morgan,10,3,14
+NL,1390,lr_morgan,1946,0.5125,Morgan,10,5,14
+NL,1390,lr_morgan,1948,0.5125,Morgan,10,5,14
+NL,1390,lr_morgan,1952,0.5125,Morgan,10,5,14
+NL,1390,lr_morgan,1956,0.5125,Morgan,10,5,14
+NL,1390,lr_morgan,1959,0.5125,Morgan,10,5,14
+NL,1390,lr_morgan,1963,0.5125,Morgan,10,5,14
+NL,1390,lr_morgan,1967,0.5125,Morgan,10,5,14
+NL,1390,lr_morgan,1971,0.5125,Morgan,10,5,14
+NL,1390,lr_morgan,1972,0.5125,Morgan,10,5,14
+NL,1581,lr_morgan,1971,0.1967,Morgan,10,2,11
+NL,1581,lr_morgan,1972,0.1967,Morgan,10,2,11
+NO,448,lr_morgan,1945,0.3529,Morgan,10,4,20
+NO,448,lr_morgan,1949,0.3529,Morgan,10,4,20
+NO,448,lr_morgan,1953,0.3529,Morgan,10,4,20
+NO,448,lr_morgan,1957,0.3529,Morgan,10,4,20
+NO,448,lr_morgan,1961,0.3529,Morgan,10,4,20
+NO,448,lr_morgan,1965,0.3529,Morgan,10,4,20
+NO,448,lr_morgan,1969,0.3529,Morgan,10,4,20
+NO,448,lr_morgan,1973,0.3529,Morgan,10,4,20
+NO,503,lr_morgan,1945,0.995,Morgan,10,10,20
+NO,503,lr_morgan,1949,0.995,Morgan,10,10,20
+NO,503,lr_morgan,1953,0.995,Morgan,10,10,20
+NO,503,lr_morgan,1957,0.995,Morgan,10,10,20
+NO,503,lr_morgan,1961,0.995,Morgan,10,10,20
+NO,503,lr_morgan,1965,0.995,Morgan,10,10,20
+NO,503,lr_morgan,1969,0.995,Morgan,10,10,20
+NO,503,lr_morgan,1973,0.995,Morgan,10,10,20
+NO,705,lr_morgan,1945,0.6907,Morgan,10,7,19
+NO,705,lr_morgan,1949,0.6907,Morgan,10,7,19
+NO,705,lr_morgan,1953,0.6907,Morgan,10,7,19
+NO,705,lr_morgan,1957,0.6907,Morgan,10,7,19
+NO,705,lr_morgan,1961,0.6907,Morgan,10,7,19
+NO,705,lr_morgan,1965,0.6907,Morgan,10,7,19
+NO,705,lr_morgan,1969,0.6907,Morgan,10,7,19
+NO,705,lr_morgan,1973,0.6907,Morgan,10,7,19
+NO,1072,lr_morgan,1945,0.7813,Morgan,10,8,20
+NO,1072,lr_morgan,1949,0.7813,Morgan,10,8,20
+NO,1072,lr_morgan,1953,0.7813,Morgan,10,8,20
+NO,1072,lr_morgan,1957,0.7813,Morgan,10,8,20
+NO,1072,lr_morgan,1961,0.7813,Morgan,10,8,20
+NO,1072,lr_morgan,1965,0.7813,Morgan,10,8,20
+NO,1072,lr_morgan,1969,0.7813,Morgan,10,8,20
+NO,1072,lr_morgan,1973,0.7813,Morgan,10,8,20
+NO,1079,lr_morgan,1945,0.005,Morgan,10,0,20
+NO,1079,lr_morgan,1949,0.005,Morgan,10,0,20
+NO,1079,lr_morgan,1953,0.005,Morgan,10,0,20
+NO,1079,lr_morgan,1957,0.005,Morgan,10,0,20
+NO,1173,lr_morgan,1945,0.6426999999999999,Morgan,10,6,20
+NO,1173,lr_morgan,1949,0.6426999999999999,Morgan,10,6,20
+NO,1173,lr_morgan,1953,0.6426999999999999,Morgan,10,6,20
+NO,1173,lr_morgan,1957,0.6426999999999999,Morgan,10,6,20
+NO,1173,lr_morgan,1961,0.6426999999999999,Morgan,10,6,20
+NO,1173,lr_morgan,1965,0.6426999999999999,Morgan,10,6,20
+NO,1173,lr_morgan,1969,0.6426999999999999,Morgan,10,6,20
+NO,1173,lr_morgan,1973,0.6426999999999999,Morgan,10,6,20
+SE,199,lr_morgan,1948,0.6456999999999999,Morgan,10,6,19
+SE,199,lr_morgan,1952,0.6456999999999999,Morgan,10,6,19
+SE,199,lr_morgan,1956,0.6456999999999999,Morgan,10,6,19
+SE,199,lr_morgan,1958,0.6456999999999999,Morgan,10,6,19
+SE,199,lr_morgan,1960,0.6456999999999999,Morgan,10,6,19
+SE,199,lr_morgan,1964,0.6456999999999999,Morgan,10,6,19
+SE,199,lr_morgan,1968,0.6456999999999999,Morgan,10,6,19
+SE,199,lr_morgan,1970,0.6456999999999999,Morgan,10,6,19
+SE,199,lr_morgan,1973,0.6456999999999999,Morgan,10,6,19
+SE,487,lr_morgan,1948,0.3251,Morgan,10,3,20
+SE,487,lr_morgan,1952,0.3251,Morgan,10,3,20
+SE,487,lr_morgan,1956,0.3251,Morgan,10,3,20
+SE,487,lr_morgan,1958,0.3251,Morgan,10,3,20
+SE,487,lr_morgan,1960,0.3251,Morgan,10,3,20
+SE,487,lr_morgan,1964,0.3251,Morgan,10,3,20
+SE,487,lr_morgan,1968,0.3251,Morgan,10,3,20
+SE,487,lr_morgan,1970,0.3251,Morgan,10,3,20
+SE,487,lr_morgan,1973,0.3251,Morgan,10,3,20
+SE,690,lr_morgan,1948,0.995,Morgan,10,10,20
+SE,690,lr_morgan,1952,0.995,Morgan,10,10,20
+SE,690,lr_morgan,1956,0.995,Morgan,10,10,20
+SE,690,lr_morgan,1958,0.995,Morgan,10,10,20
+SE,690,lr_morgan,1960,0.995,Morgan,10,10,20
+SE,690,lr_morgan,1964,0.995,Morgan,10,10,20
+SE,690,lr_morgan,1968,0.995,Morgan,10,10,20
+SE,690,lr_morgan,1970,0.995,Morgan,10,10,20
+SE,690,lr_morgan,1973,0.995,Morgan,10,10,20
+SE,830,lr_morgan,1948,0.005,Morgan,10,0,20
+SE,830,lr_morgan,1952,0.005,Morgan,10,0,20
+SE,830,lr_morgan,1956,0.005,Morgan,10,0,20
+SE,830,lr_morgan,1958,0.005,Morgan,10,0,20
+SE,830,lr_morgan,1960,0.005,Morgan,10,0,20
+SE,830,lr_morgan,1964,0.005,Morgan,10,0,20
+SE,830,lr_morgan,1968,0.005,Morgan,10,0,20
+SE,830,lr_morgan,1970,0.005,Morgan,10,0,20
+SE,830,lr_morgan,1973,0.005,Morgan,10,0,20
+SE,1274,lr_morgan,1948,0.7444,Morgan,10,7,20
+SE,1274,lr_morgan,1952,0.7444,Morgan,10,7,20
+SE,1274,lr_morgan,1956,0.7444,Morgan,10,7,20
+SE,1274,lr_morgan,1958,0.7444,Morgan,10,7,20
+SE,1274,lr_morgan,1960,0.7444,Morgan,10,7,20
+SE,1274,lr_morgan,1964,0.7444,Morgan,10,7,20
+SE,1274,lr_morgan,1968,0.7444,Morgan,10,7,20
+SE,1274,lr_morgan,1970,0.7444,Morgan,10,7,20
+SE,1274,lr_morgan,1973,0.7444,Morgan,10,7,20

data/morgan_positions_raw.csv

@@ -0,0 +1,161 @@
+country,period,party_abbrev,party_name,position,n_surveys,sd
+DNK,1919-1939,SOCd,Social Democrats,0.0,19,0.0
+DNK,1919-1939,RAD,Radicals,33.94,19,4.62
+DNK,1919-1939,LIB,Liberals,71.87,19,7.99
+DNK,1919-1939,CONS,Conservatives,100.0,19,0.0
+DNK,1945-1973,COMM,Communists,0.0,22,0.0
+DNK,1945-1973,LS,Left Socialists,6.5,22,4.17
+DNK,1945-1973,SOCL,Socialists (SF),19.54,22,5.93
+DNK,1945-1973,SOCd,Social Democrats,41.49,22,7.19
+DNK,1945-1973,RAD,Radicals,61.85,22,6.23
+DNK,1945-1973,LC,Liberal Center,67.9,18,8.27
+DNK,1945-1973,LIB,Liberals,81.98,22,6.46
+DNK,1945-1973,JUST,Justice Party,84.08,17,13.38
+DNK,1945-1973,CONS,Conservatives,100.0,22,0.0
+FIN,1919-1939,SKDL,Communist,0.0,12,0.0
+FIN,1919-1939,SOCd,Social Democrats,24.92,12,2.8
+FIN,1919-1939,PROG,National Progressives,57.5,12,4.68
+FIN,1919-1939,AGR,Agrarian Union,59.32,12,4.46
+FIN,1919-1939,SWPP,Swedish People's Party,67.79,12,6.57
+FIN,1919-1939,CONS,Finnish Conservative Party,91.96,12,3.9
+FIN,1919-1939,NPF,Finnish Patriotic Movement,100.0,12,0.0
+FIN,1945-1973,PDEM,Democratic League,0.0,15,0.0
+FIN,1945-1973,SDWS,Workers Smallholders SD League,21.15,15,6.73
+FIN,1945-1973,SOCd,Social Democrats,25.78,15,3.41
+FIN,1945-1973,CENT,Center Party,57.76,13,4.39
+FIN,1945-1973,FRP,Finnish Rural Party,64.11,13,16.91
+FIN,1945-1973,LIB,Finnish People Party/Liberal League,70.34,15,5.5
+FIN,1945-1973,SWPP,Swedish People's Party,79.75,15,6.23
+FIN,1945-1973,CONS,Finnish Conservative Party,98.55,15,2.89
+ISL,1919-1939,COMM,Communist People Alliance,0.0,6,0.0
+ISL,1919-1939,SOCd,Social Democratic Party,21.01,6,4.25
+ISL,1919-1939,PROG,Progressives/Farmers Party,62.11,6,12.23
+ISL,1919-1939,LIB,Liberal Party,63.02,6,10.81
+ISL,1919-1939,INDP,Independence Party,81.28,6,5.23
+ISL,1919-1939,CONS,Conservative Party,100.0,6,0.0
+ISL,1945-1973,COMM,Communist People Alliance,0.0,9,0.0
+ISL,1945-1973,LLIB,Liberal Left,11.41,9,3.89
+ISL,1945-1973,SOCd,Social Democratic Party,37.78,9,8.8
+ISL,1945-1973,PROG,Progressive Party,72.99,9,7.7
+ISL,1945-1973,INDP,Independence Party,100.0,9,0.0
+NOR,1919-1939,LAB,Labor Party,0.0,19,0.0
+NOR,1919-1939,LIB,Liberal Party,57.71,19,7.47
+NOR,1919-1939,AGR,Agrarian Party,70.61,19,8.64
+NOR,1919-1939,CONS,Conservative Party,100.0,19,0.0
+NOR,1945-1973,COMM,Communist Party,0.0,20,0.0
+NOR,1945-1973,SOCL,Socialist People Party,7.45,20,4.16
+NOR,1945-1973,LAB,Labor Party,35.29,20,5.05
+NOR,1945-1973,LIB,Liberal Party,64.27,20,6.84
+NOR,1945-1973,CHPP,Christian People Party,69.07,19,8.84
+NOR,1945-1973,CENT,Center Party,78.13,20,0.0
+NOR,1945-1973,CONS,Conservative Party,100.0,20,0.0
+SWE,1919-1939,COMM,Communist Party,0.0,15,0.0
+SWE,1919-1939,SOCd,Social Democratic Party,25.85,15,5.38
+SWE,1919-1939,AGR,Agrarian Party,58.57,15,5.43
+SWE,1919-1939,LIB,Liberal Party,74.19,15,4.41
+SWE,1919-1939,CONS,Conservative Party,100.0,15,0.0
+SWE,1945-1973,COMM,Communist Party,0.0,20,0.0
+SWE,1945-1973,SOCd,Social Democratic Party,32.51,20,6.17
+SWE,1945-1973,CENT,Center/Agrarian Party,64.57,19,3.34
+SWE,1945-1973,LIB,Liberal/People Party,74.44,20,6.2
+SWE,1945-1973,CONS,Conservative Party,100.0,20,0.0
+NLD,1919-1939,CPN,Communist Party,0.0,6,0.0
+NLD,1919-1939,SOCd,Social Democratic Party (SDAP),18.32,6,5.41
+NLD,1919-1939,RAD,Radical Party,47.04,6,3.14
+NLD,1919-1939,KVP,Catholic Party,56.05,6,6.44
+NLD,1919-1939,CHU,Christian Historical Union,69.2,6,7.59
+NLD,1919-1939,ULIB,United Liberal Party,69.7,6,3.13
+NLD,1919-1939,FLIB,Free Liberal Party,70.83,6,6.14
+NLD,1919-1939,LIB,Liberal Party,71.94,6,4.1
+NLD,1919-1939,ARP,Anti-Revolutionary Party,72.43,6,9.1
+NLD,1919-1939,ECUM,Ecumenical Party,76.25,2,19.45
+NLD,1919-1939,SGP,Political Reformed Party,88.5,6,1.32
+NLD,1919-1939,NSB,National Social Movement,100.0,6,0.0
+NLD,1945-1973,CPN,Communist Party,0.0,14,0.0
+NLD,1945-1973,PSP,Pacifist Socialist Party,7.66,14,4.65
+NLD,1945-1973,PPR,Political Radicals,19.67,11,4.98
+NLD,1945-1973,PVDA,Labor Party,26.69,14,4.91
+NLD,1945-1973,D66,Radical Democrats,39.11,13,4.06
+NLD,1945-1973,DS70,Social Democratic splinter,46.04,14,6.15
+NLD,1945-1973,KVP,Catholic Party,51.25,14,2.08
+NLD,1945-1973,ARP,Anti-Revolutionary Party,55.56,14,8.58
+NLD,1945-1973,CHU,Christian Historical Union,66.47,14,5.65
+NLD,1945-1973,VVD,Liberal Party,76.1,14,5.82
+NLD,1945-1973,SGP,Political Reform Party,96.35,13,3.84
+NLD,1945-1973,GPV,Reformed Political Association,96.7,12,3.88
+NLD,1945-1973,BP,Peasant Party (Poujadist),96.82,11,4.46
+BEL,1919-1939,COMM,Communist Party,0.0,11,0.0
+BEL,1919-1939,POB,Social Democratic Party,25.11,11,5.48
+BEL,1919-1939,CATH,Catholic Party,67.85,11,9.78
+BEL,1919-1939,LIB,Liberal Party,78.84,10,8.27
+BEL,1919-1939,FNAT,Flemish Nationalist parties,80.78,11,11.93
+BEL,1919-1939,REX,Rexist,98.72,10,3.14
+BEL,1945-1973,COMM,Communist Party,0.0,17,0.0
+BEL,1945-1973,PSB,Socialist Party,31.93,17,5.25
+BEL,1945-1973,RW,Rassemblement Wallon,53.65,16,9.73
+BEL,1945-1973,PSC,Christian Social Party,70.9,17,6.59
+BEL,1945-1973,FDF,Front Democratique Wallon,71.89,17,13.85
+BEL,1945-1973,VOLK,Volksunie,83.21,17,6.64
+BEL,1945-1973,PLP,Liberal Party,100.0,17,0.0
+FRA,1946-1958,PCF,Communist Party,0.0,23,0.0
+FRA,1946-1958,SFIO,Socialist Party,23.65,23,4.61
+FRA,1946-1958,MRP,Popular Republican Movement,43.12,23,6.73
+FRA,1946-1958,RDA,Radical Party,43.37,20,6.73
+FRA,1946-1958,UDSR,Democratic Socialist Union,44.62,20,6.68
+FRA,1946-1958,RAD,Radical Democratic Assembly,50.94,22,5.3
+FRA,1946-1958,RS,Social Republicans,66.52,23,9.15
+FRA,1946-1958,RPF,Rally of the French People,70.0,23,7.25
+FRA,1946-1958,AR,Republican Action,72.93,20,3.74
+FRA,1946-1958,ARS,Republican Social Action,77.38,16,6.83
+FRA,1946-1958,RI,Independent Republicans,82.62,19,8.05
+FRA,1946-1958,CNIP,National Center of Independents,83.5,19,5.1
+FRA,1946-1958,PUS,Pro-Gaullist faction,85.0,10,5.14
+FRA,1946-1958,PAYS,Peasant Party,85.23,14,6.55
+FRA,1946-1958,AP,Popular Action,85.4,11,4.34
+FRA,1946-1958,PRL,Republican Liberty Party,87.56,17,5.66
+FRA,1946-1958,POUJ,Poujadists,100.0,22,0.0
+DEU,1919-1930,KPD,Communist Party,0.0,15,0.0
+DEU,1919-1930,SDAP,Social Democratic Party,25.58,15,4.62
+DEU,1919-1930,DDP,Democratic Party,45.57,15,4.79
+DEU,1919-1930,DZP,Center Party,51.25,15,6.35
+DEU,1919-1930,BVP,Bavarian People's Party,63.17,11,5.63
+DEU,1919-1930,DVP,German People's Party,65.17,15,4.79
+DEU,1919-1930,RDMW,Economic Party,70.64,11,5.93
+DEU,1919-1930,LVP,Rural People's Party,80.35,9,6.53
+DEU,1919-1930,DNVP,German National People's Party,88.17,15,3.68
+DEU,1919-1930,NAZI,National Socialist Party,100.0,15,0.0
+ITA,1946-1975,PCI,Communist Party,3.08,12,3.37
+ITA,1946-1975,PSIU,Socialist Party of Proletarian Unity,3.61,12,4.0
+ITA,1946-1975,PSI,Socialist Party (Nenni),22.81,12,3.11
+ITA,1946-1975,PSDI,Social Democratic Party,36.12,12,3.77
+ITA,1946-1975,PRI,Republican Party,42.18,12,2.37
+ITA,1946-1975,DC,Christian Democratic Party,57.16,12,5.44
+ITA,1946-1975,PLI,Liberal Party,72.92,12,3.02
+ITA,1946-1975,MON,Monarchists,91.68,12,2.41
+ITA,1946-1975,MSI,Italian Social Movement,100.0,12,0.0
+LUX,1945-1973,COMM,Communist Party,0.0,7,0.0
+LUX,1945-1973,SOCd,Socialist Party,36.22,7,12.83
+LUX,1945-1973,CSOC,Christian Social Party,85.76,7,13.47
+LUX,1945-1973,GRPD,Democratic Group,93.33,5,11.55
+ISR,1949-1973,RAKA,Rakah,0.0,10,0.0
+ISR,1949-1973,MAKI,Maki,3.46,10,2.79
+ISR,1949-1973,MAPM,Mapam,18.04,10,5.85
+ISR,1949-1973,MADT,Mapam Ahdut Haavoda,26.78,10,9.24
+ISR,1949-1973,ADUT,Ahdut Haavoda,34.92,10,12.63
+ISR,1949-1973,MAAR,Maarach,46.79,10,8.39
+ISR,1949-1973,LAB,Labor Alignment,48.08,10,6.65
+ISR,1949-1973,MAPI,Mapai,48.27,10,6.99
+ISR,1949-1973,PAUG,Poalei Agudat,55.82,6,5.24
+ISR,1949-1973,RAFI,Rafi,60.42,7,6.71
+ISR,1949-1973,PROG,Progressives,65.0,8,5.38
+ISR,1949-1973,ILIB,Independent Liberals,65.8,9,7.55
+ISR,1949-1973,NRP,National Religious Party,69.0,7,8.21
+ISR,1949-1973,URF,United Religious Front,77.0,5,10.63
+ISR,1949-1973,LIB,Liberals,78.57,9,10.35
+ISR,1949-1973,NATL,National List,79.67,6,10.87
+ISR,1949-1973,TORA,Torah,85.29,6,0.67
+ISR,1949-1973,LIKD,Likud,87.39,6,11.04
+ISR,1949-1973,ZION,General Zionists,87.95,9,3.35
+ISR,1949-1973,GHAL,Gahal,90.69,8,4.71
+ISR,1949-1973,AGDT,Agudat Israel,96.25,5,2.75
+ISR,1949-1973,HRUT,Herut,98.0,9,2.71

data/party_names.csv

@@ -0,0 +1,732 @@
+partyfacts_id,party_name_english,party_name_short
+3,Social Democratic Party,SDP
+4,We Ourselves,SF
+6,Liberal-Socialists of Chile,UCC
+10,The Greens,B90/Gruene
+14,Radical Party,PR
+18,Estonian United People’s Party,EÜRP
+29,Social Democratic Party of Switzerland,SPS/PSS
+34,Communist Party,PCI
+42,Movement for a Better Hungary,Jobbik
+43,List Dedecker,LDD
+45,Democrats 66,D’66
+48,Communist Party of Greece,KKE
+49,Open Flemish Liberals and Democrats,openVLD
+50,South Schleswig Voters’ Union,SSW
+53,Christian Democrats,K
+54,Party for Democracy,PPD
+57,Democratic Left Alliance,SLD
+59,Democratic Change,CD
+63,Christian Democratic Movement,KDH
+64,Lithuanian Social Democratic Party,LSDP
+72,Croatian Peasant Party,HSS
+78,Democratic Centre,DC
+81,Canarian Coalition-New Canaries,CCa-PNC-NC
+82,Union of the Democratic Centre,EDIK
+86,The Left. Party of Democratic Socialism,L-PDS
+96,Slovenian National Party,SNS
+97,People-Animals-Nature,PAN
+101,Progress Party,FrP
+102,Labour Party,DP
+109,Worker's Party of the Land of Israel,Mapai
+110,Estonian People's Union,ERL
+118,National Party,PN
+120,Social Democratic Party,PSD
+150,Progressive Party,Progressive
+152,New Democratic Party,NDP
+155,People's Democratic Party,PDP
+162,Communist Party of Chile,PC
+163,Anti-Revolutionary Party,ARP
+169,United Workers' Party,MAPAM
+171,Liberal Democratic Party,LDP
+172,Reformatory Political Federation,RPF
+176,For Human Rights in a United Latvia,PCTVL
+177,Czech Social Democratic Party,ČSSD
+190,Democratic Party of Albania,PD
+193,Homeland Union,TS
+194,Social Christian Reformist Party,PRSC
+199,Centre Party,CP
+208,Workers' Party,WP
+209,Humanist Party,PH
+212,Liberal Alliance,NY
+214,Croatian Social-Liberal Party,HSLS
+216,Democratic Revolutionary Party,PRD
+223,"Tradition, Responsibility, Prosperity 09",TOP09
+225,Party of Brazilian Social Democracy,PSDB
+232,Progressive Conservative Party,PCP
+235,Israel is Our Home,YB
+237,Order and Justice,PTT
+246,Patriotic Society Party,PSP
+247,United Left,IU
+249,Liberal People's Party,LKP
+253,Croatian Party of Rights,HSP
+257,National Restoration Party,PRN
+258,Bridge,Most-Hid
+259,Italian Renewal,RI
+263,Bulgarian People's Union,BNS
+271,The Left,DL
+275,Party of Liberty and Progress,PVV/PLP
+276,Social Democratic Party,PSD
+277,Danish Communist Party,DKP
+279,Daisy - Democracy is Freedom,M-DL
+281,Walloon Rally,RW
+285,Liberal National Party of Queensland,LNP
+292,Alliance of Independent Social Democrats,SNSD
+298,Party of Freedom,PVV
+300,Democratic Party,DP/PD
+304,Christian Democratic Union,KDS
+306,Justice and Development Party,AKP
+308,Swiss People's Party,SVP/UDC
+310,Portuguese Communist Party,PCP
+311,Direction-Social Democracy,Smer
+312,Political Spring,Pola
+329,Socialist People's Party,SF
+335,Democratic Unionist Party,DUP
+338,Nationalist Party of Australia,NAT
+340,Communist Party of Slovakia,KSS
+349,Party of the Hungarian Coalition,SMK/MKP
+351,True Path Party,DYP
+352,People's New Party,PNP
+356,Workers' Party,PT
+357,Party for Bosnia and Herzegovina,SBiH
+362,Colombian Liberal Party,PLC
+363,Independence Party,Sj
+378,Ecologist Greens,OP
+379,Social Democratic Party,SD
+382,Liberal Party,PL
+383,Social Democratic Party of Germany,SPD
+390,Christian Democratic Party,PDC
+394,Italian Republican Party,PRI
+405,Francophone Christian Social Party and Flemish Christian People's Party,PSC/CVP
+409,Sweden Democrats,SD
+421,Democratic Front for Peace and Equality,HADASH
+424,Australian Labor Party,ALP
+431,Democratic Party of Albanians,DPA/PDSh
+432,Democratic Party,Democrats
+433,National Front,FN
+434,Progressive and Independent Liberal Party,ILP
+437,Social Democrat Radical Party,PRSD
+441,People's Party,PP
+442,Movement towards Socialism - Political Instrument for the Sovereignty of the Peoples,MAS-IPSP
+444,Liberal Party of Switzerland,LPS/PLS
+446,Mexican Green Ecologist Party,PVEM
+447,United Arab List,Ra'am
+448,Labour Party,Ap
+455,Sephardi Torah Guardians,Shas
+456,Green Ecology Party,MP
+457,Left Green Movement,VGF
+458,Brazilian Labour Party,PTB
+459,Communist Party of the Netherlands,CPN
+463,Austrian Freedom Party,FPÖ
+466,Civic Democratic Party,ODS
+467,Democratic Party of Pensioners of Slovenia,DeSUS
+469,Democratic Coalition,DK
+472,Slovenian Democratic Party,SDS
+474,For Real,Zares
+479,Green Union,VL
+480,Belgian Socialist Party,BSP/PSB
+481,National Liberal Party,PNL
+484,Forward,Kadima
+487,Social Democratic Labour Party,SAP
+491,Res Publica Party,ERP
+492,Social Democratic Federation,SDF
+495,National Coalition,KK
+496,Democratic Movement,MoDem
+500,Francophone Socialist Party,PS
+503,Conservative Party,H
+506,National Alliance 'All For Latvia! - For Fatherland and Freedom/LNNK',NA
+509,Rally for the Republic,RPR
+513,Istrian Democratic Assembly,IDS
+514,Popular Action,AP
+516,Progressive Left Coalition,SYN
+525,Women's Alliance,Kv
+528,Green!,Groen!
+529,Peruvian Aprista Party,PAP
+533,Estonian Center Party,K
+536,Conservative People's Party,KF
+537,Social Democratic Party of Bosnia and Herzegovina,SDP BiH
+539,Christian Social People's Party,CSV/PCS
+540,Liberal Party,Liberal
+546,Lithuanian Democratic Labour Party,LDDP
+553,Flemish Bloc,VB
+554,Liberal Reformation Party,PRL
+556,Election Action of Lithuania's Poles,LLRA/AWPL
+557,Change,SHINUI
+560,Movement for a Democratic Slovakia,HZDS
+561,Party of Radical Change,PCR
+562,Labour Party,Labour
+563,Democratic Coalition,DISY
+573,Free Democratic Party,FDP
+575,Union for Peru,UPP
+591,New Zealand First Party,NZF
+594,Democratic Alliance Party,AD
+598,Proletarian Unity Party for Communism The Manifesto + Proletarian Unity Party,PdUP
+599,Alliance for the Future of Austria,BZÖ
+601,United Kingdom Independence Party,UKIP
+602,Polish Peasants' Party,PSL
+605,Liberal Forum,LIF
+614,Democratic Alliance,AD
+615,Alignment,AMT
+622,Christian Democratic and Flemish,CD&V
+623,Justicialist Party,PJ
+631,Federal Democratic Union,EDU/UDF
+633,Humanist Democratic Centre,cdH
+635,Democratic Party,DiKo
+638,Renaissance Movement,Tehiya
+641,Union of Israel,AI
+644,Slovenian Christian Democrats,SKD
+645,Party of the Civic Alliance,PAC
+651,Christian Democrats,Kd
+655,Socialist Party,PS
+656,Kotleba – People's Party Our Slovakia,ĽSNS
+669,Swiss Labour Party,PdAS/PdTS
+671,For Fatherland and Freedom,TB
+674,United Albanian Right,DBSH
+690,Moderate Coalition Party,MSP
+696,Alliance for Change,PAN
+698,"Democratic, Federalist, Independent",DéFI
+699,Nationalist Party,PN
+703,Christian Democrats in Finland,KD
+705,Christian People's Party,KrF
+714,50Plus,50PLUS
+719,Socialist Left Party,SV
+722,Navarrese People's Union,UPN
+723,Brazilian Socialist Party,PSB
+724,Socialist Party of Macedonia,SPM
+727,Self-Defence of the Polish Republic,SRP
+730,Slovak National Party,SNS
+732,Legality Movement Party,PLL
+736,Japanese Communist Party,JCP
+737,Popular Republican Movement,MRP
+741,Latvian Social Democratic Alliance,LSDA
+742,Italian Socialist Party,PSI
+745,Crossroads - non-aligned movement for Zionist Renewal,Tzomet
+752,ACT New Zealand,ACT
+756,Christian-Democrat and Flemish / New Flemish Alliance,CD+NVA
+757,Bulgarian Socialist Party,BSP
+759,Party of U,PU
+760,Citizens for European Development of Bulgaria,GERB
+762,Social Democratic and Labour Party,SDLP
+764,Slovenian People's Party,SLS
+768,List Di Pietro - Italy of Values,IdV
+789,Reform Movement,MR
+794,New Democracy,ND
+797,Democratic Left,ID
+799,Croatian People's Party - Liberal Democrats,HNS
+800,Centre Democrats,CD
+802,Democratic Party,PD
+808,Labour Party,PL
+809,Republican Party,Republicans
+813,National Alliance,AN
+815,National Salvation Party,MSP
+819,Albanian Republican Party,PRSH
+821,Estonian Reform Party,ER
+824,Christian Democratic Party,KDS
+827,Pachakutik Plurinational Unity Movement - New Country,MUPP-NP
+828,People's Party for Freedom and Democracy,VVD
+830,Left Party,V
+838,Social Democratic Party of Albania,PSD
+840,Agrarian Party of Albania,PASH
+842,Libertarian Movement Party,PML
+848,Catalan Republican Left,ERC
+851,Green Federation,FdV
+858,Ecological and Environmental Movement,KOP
+863,Electoral Action ‘Solidarity',AWS
+867,Democratic Socialist Party,DSP
+878,Democrats of the Left,DS
+886,Ecologist Party 'The Greens',PEV
+888,New Italian Socialist Party,NPSI
+894,Democratic Liberal Party,DLP
+896,Popular Democratic Union,UDP
+898,Social Democratic Party,SDP
+901,Finnish Centre,SK
+907,New Communist Party,RAKAH
+908,"National Coalition Party ""Pro Patria""",RKI
+910,Alliance of Free Democrats,SzDSz
+911,New Zealand Democratic Party,NZDP
+919,Democratic Party,SD
+921,Democratic Socialists '70,DS‘70
+924,Progressive Democrats,PD
+928,National Renewal,RN
+931,Green Party,GPC
+932,Independents' Party,DU
+934,Italian Popular Party,PPI
+940,United Future New Zealand,UF
+946,Aragonese Party,PAR
+949,Solidarity Party,PS
+950,Social Christian Party,PSC
+952,Movement for an Autonomous Democracy - Society for Moravia-Silesia,HSD-SMS
+953,National Union,IL
+962,Christian Democratic Centre / United Christian Democrats,CCD/CDU
+963,We Ourselves,SF
+964,Progressive Party,F
+965,Popular Orthodox Rally,LAOS
+975,Liberal Democracy of Slovenia,LDS
+982,Movement for Rights and Freedoms,DPS
+983,Democratic Party,DS
+986,Scottish National Party,SNP
+987,Christian Democratic Party,PDC
+990,Australian Democrats,AD
+1002,The Party of Wales,PC
+1004,Conservative Party of Canada,CP
+1006,Green Party of Switzerland,GPS/PES
+1008,Civic Democratic Alliance,ODA
+1009,Democratic Labour Party,PDT
+1011,Galician Nationalist Bloc,BNG
+1013,National Popular Front,ELAM
+1022,Danish People's Party,DF
+1024,Party of the Land,CnT
+1027,Bulgarian Business Bloc,BBB
+1028,Social Democrats' Movement,EDEK
+1033,Party for Democratic Prosperity,PDP
+1038,Democratic Alliance,DA
+1040,Estonian Greens,EER
+1043,Latvian Way Union,LC
+1044,Left Wing Alliance,VAS
+1049,New Zealand Labour Party,Labour
+1053,Israel for Immigration,YBA
+1055,Soldiers of Destiny,FF
+1056,Concord Centre,SC
+1067,Welfare Party,RP
+1072,Centre Party,Sp
+1075,United Torah Judaism,UTJ
+1076,Progressive Party of the Working People,AKEL
+1077,Coexistence,ESWS
+1079,Norwegian Communist Party,NKP
+1083,Union for a New Majority - Gaullists/Conservatives,RPF
+1096,Finnish People's Democratic Union,SKDL
+1099,Green Party of Aotearoa New Zealand,Greens
+1102,Christian Historical Union,CHU
+1104,Union of Labour,UP
+1105,Hungarian Democratic Alliance of Romania,UDMR/RMDSz
+1114,Alternative Democratic Reform Party,ADR
+1119,New Labour Party,NLP
+1123,Protestant People's Party of Switzerland,EVP/PEV
+1126,Italian Democratic Socialist Party,PSDI
+1131,Institutional Renewal Party of National Action,PRIAN
+1134,Justice Party,RF
+1136,Left Socialist Party,VS
+1138,Green Alternative,GAP
+1150,Social Democratic Party,SDE
+1156,Social Credit,Socred
+1157,Christian Democratic Appeal,CDA
+1160,Democratic Left,DIMAR
+1163,Democratic Social Party,PDS
+1164,Social Democratic League of Workers and Smallholders,TPSL
+1166,Serbian Democratic Party,SDS
+1173,Liberal Party,V
+1178,Reformed Political Party,SGP
+1183,United Democratic Forces,ODS
+1185,Lithuanian Centre Union,LCS
+1190,Motherland Party,ANAP
+1193,Liberal and Centre Union,LiCS
+1194,Independents' Alliance,LdU/AdI
+1195,Democrats for a Strong Bulgaria,DSB
+1199,Social Christian Unity Party,PUSC
+1204,Liberals,V
+1209,Australian Greens,Greens
+1215,Alternative for a Republic of Equals,ARI
+1219,African National Congress,ANC
+1220,Democratic Union for Integration,DUI/BDI
+1221,League,L
+1224,Liberal Party,FF
+1225,Dominican Liberation Party,PLD
+1229,Swedish People's Party,RKP/SFP
+1231,FDP. The Liberals,FDP/PLR
+1234,Labour Party,PvdA
+1241,Labor Party,PT
+1242,Party of the Democratic Left,SDľ
+1246,Independent Republicans | Liberal Democracy,IR|DL
+1247,Citizen's Action Party,PAC
+1248,Centre Democrats,CDS
+1249,People's Alliance,Ab
+1251,French Communist Party,PCF
+1252,Broad Front,FA
+1253,Social Democratic Populist Party,SHP
+1271,Union of the Democratic Centre/Centrist Bloc,UCD
+1274,Liberals,L
+1287,Pirates,Pi
+1288,Family of the Irish,FG
+1294,Independent Smallholders' Party,FKgP
+1299,Estonian National Independence Party,ERSP
+1303,Finnish Social Democrats,SSDP
+1305,Greater Romania Party,PRM
+1310,Left Bloc,BE
+1311,Freedom Party of Switzerland,FPS
+1324,Basque Solidarity,EA
+1325,Social Democratic Party,A
+1326,Basque Left,EE
+1329,Austrian People's Party,ÖVP
+1331,Citizens' Movement,CD
+1338,Spanish Socialist Workers' Party,PSOE
+1345,New Alliance Party,PANAL
+1351,Democratic Revolutionary Party,PRD
+1354,Republican Party,CnP
+1357,Lithuanian Liberal Union,LLS
+1358,Citizens' Rights Movement,RATZ
+1359,Social Democratic Party,PSD
+1363,Socialist Party,SP
+1366,Ulster Unionist Party,UUP
+1367,National Liberation Party,PLN
+1369,South Tyrolean People’s Party,SVP
+1373,Romanian Ecological Party,PER
+1375,Christian Democratic Union,CDU
+1384,Austrian Social Democratic Party,SPÖ
+1386,Freedom and Solidarity,SaS
+1388,Liberal Democrats,LibDems
+1390,Catholic People's Party,KVP
+1396,The Alliance - Social Democratic Party of Iceland,S
+1398,Unity of Labour,AHA
+1403,Social Democratic Party,SD
+1404,Communist Refoundation Party,PRC
+1405,Popular Democracy - Christian Democratic Union,DP - UDC
+1408,Hungarian Socialist Party,MSzP
+1412,Christian Democratic People's Party,KDNP
+1415,Conservative Democratic Party of Switzerland,BDP/PBD
+1417,Israel Communist Party,Maki
+1419,Colorado Party,PC
+1424,Peoples' Union,VU
+1428,Quebec Bloc,BQ
+1431,Croatian Democratic Union,HDZ
+1436,Israeli Labour Party,HaAvoda
+1439,German Minority,MN
+1441,Dominican Revolutionary Party,PRD
+1447,National Religious Party,Mafdal
+1448,Socialist Party,SP
+1449,Pannella-Sgarbi List,PR
+1450,Ecuadorian Roldosist Party,PRE
+1451,Romanian National Unity Party,PUNR
+1454,Party of Democratic Action,SDA
+1459,Christian Union,CU
+1461,Italian Liberal Party,PLI
+1467,Party for the Animals,PvdD
+1472,Centre Democrats,CD
+1474,Institutional Revolutionary Party,PRI
+1478,Socialist Party,PS
+1479,Democratic Party of Slovenia,DSS
+1480,Popular Monarchist Party,PPM
+1489,General Zionists,GZ
+1490,Lithuanian Peasant and Green Union,LVŽS
+1494,Greens of Slovenia,ZS
+1497,Andalusian Party,PA
+1502,German Party,DP
+1507,Danish Social-Liberal Party,RV
+1508,Social-Democratic League of Macedonia,SDSM
+1515,New Clean Government Party,NKP
+1516,Labour Party,Labour
+1517,Union of Liberals and Leftists,SFVM
+1519,Centre Right,CeD
+1524,New Liberal Club,NLC
+1527,Red-Green Unity List,EL
+1528,List Pim Fortuyn,LPF
+1531,New Era,JL
+1533,The Greens,Groen
+1540,Democratic Labor Party,DLP
+1545,The Left,LINKE
+1554,Green Party,SZ
+1556,Coalition Party,KE
+1563,Ecologists,ECOLO
+1564,Aragonist Council,CHA
+1565,Law and Justice,PiS
+1567,Conservative Party,Conservatives
+1577,Colombian Conservative Party,PCC
+1581,Radical Political Party,PPR
+1586,Socialist Party Different - Spirit,sp.a-SPIRIT
+1597,Hungarian Justice and Life Party,MIÉP
+1599,Independent Democratic Union,UDI
+1600,Democratic Renewal Party,PRD
+1601,Progress Party,FP
+1602,Reformed Political League,GPV
+1610,National Action Party,MHP
+1615,Confederation for an Independent Poland,KPN
+1618,New Slovenian Christian People's Party,Nsi
+1626,Go Italy,FI
+1629,Union for a New Majority - Conservatives/Gaullists,PRL
+1630,Inkatha Freedom Party,IFP
+1635,Party of Italian Communists,PdCI
+1636,Union for Human Rights Party,PBDNJ
+1637,Basque Nationalist Party,PNV/EAJ
+1647,Communist Party of Luxembourg,KPL/PCL
+1650,Politics Can Be Different,LMP
+1651,Independent Greeks,ANEL
+1655,Freedom,Herut
+1657,Association for the Republic - Republican Party of Czechoslovakia,SPR-RSČ
+1659,The Greens,GRÜNE
+1660,Golden Dawn,XA
+1661,Ticino League,LdT
+1663,National Democratic Assembly,BaLaD
+1664,Homeland,Moledet
+1671,Amaiur,Amaiur
+1673,Party of Democratic Progress of the Republika Srpska,PDP RS
+1682,Democratic Social Movement,DIKKI
+1688,Popular Democratic Movement,MDP
+1689,True Finns,PS
+1691,Alliance of Federation of Young Democrats - Christian Democratic People's Party,FiDeSz-KDNP
+1696,National Alliance,AN
+1697,Hungarian Democratic Forum,MDF
+1702,Latvian Farmers' Union,LZS
+1705,United People Alliance,APU
+1715,Democratic Party,PD
+1716,Maori Party,Maori
+1719,Harmony for Latvia - Rebirth of the Economy,SLAT
+1726,Hungarian Social Democratic Party,MSzDP
+1728,Communist Party of Bohemia and Moravia,KSČM
+1729,Albanian Socialist Party,PSS
+1731,Christian Social Union,CSU
+1739,Liberal Party of Canada,LP
+1740,Democratic Left Party,DSP
+1744,Liberal Movement,LRLS
+1746,Liberal Democratic Party,LDP
+1750,Christian Democratic National Peasants’ Party,PNŢCD
+1753,Workers' Party of Belgium,PTB/PVDA
+1755,People's Party,TP
+1757,Reform Party of Canada,RPC
+1758,Union / Centre,UC
+1759,Green Liberal Party,GLP
+1767,Christian Democratic Centre,CCD
+1768,League of Polish Families,LPR
+1775,Green Party,Greens
+1777,Your Party,YP
+1783,Left Radical Party,PRG
+1789,Latvian National Independence Movement,LNNK
+1790,Alliance Union-PRO,PRO
+1793,National Union Attack,ATAKA
+1794,Green Party of England and Wales,GPEW
+1800,New Union Social Liberals,NS
+1804,Democratic Party of Japan,DPJ
+1808,Christian Democratic People's Party of Switzerland,CVP/PDC
+1816,Alliance 90/The Greens,B90/Gruene
+1817,European Party,EVROKO
+1819,Swiss Democrats,SD/DS
+1823,Liberal Party,PL
+1824,New Zealand National Party,National
+1963,Green Party,MDG
+1966,United National Party,UNP
+1968,Flemish Interest,VB
+1970,The New Austria and Liberal Forum,NEOS
+1976,Alternative for Germany,AfD
+1989,Authentic Party of the Mexican Revolution,PARM
+1991,Azerbaijan National Independence Party,AMİP
+1992,Azerbaijan Popular Front,AXC
+1995,New Azerbaijan Party,YAP
+1997,Katter's Australian Party,KAT
+1998,Liberal Party,LP
+2007,Rule of Law,OEK
+2009,National Democratic Union,AZhM
+2013,Republican Party of Armenia,HHK
+2018,Communist Party of Armenia,HKK
+2020,Armenian Revolutionary Federation,HHD
+2030,Party of Communists of Belarus,PKB
+2046,Five Star Movement,M5S
+2047,Czech Pirate Party,Piráti
+2048,Pirate Party,Þ
+2057,National Front for the Salvation of Bulgaria,NFSB
+2080,Japan Innovation Party,JIP
+2130,Ordinary People and Independent Personalities,OľaNO
+2141,ANO 2011,ANO
+2148,Citizens’ Union of Georgia,SMK
+2153,Georgian Labour Party,SLP
+2159,Industry will save Georgia,MGS
+2161,National Democratic Party,EDP
+2162,New Rights,AM
+2168,Union for the Democratic Revival,DAP
+2169,Union of Georgian Traditionalists,KTK
+2172,United National Movement,ENM
+2175,Serbian Radical Party,SRS
+2176,Serbian Renewal Movement,SPO
+2178,Socialist Party of Serbia,SPS
+2186,Democratic Community of Magyars of Vojvodina,DZVM
+2189,Democratic Party,DS
+2193,G17plus,G17+
+2196,Liberal Democratic Party,LDP
+2197,New Democracy,ND
+2199,Party for Democratic Activity,PVD/PDD
+2203,Alliance of Vojvodina Hungarians,VMSZ
+2207,Popular Movement of Ukraine,Rukh
+2210,Social Democratic Party of Ukraine,SDPU
+2211,Bloc Socialist Party of Ukraine and Peasant Party of Ukraine,SPU-SelPU
+2220,Communist Party of Ukraine,KPU
+2228,Juliya Tymoshenko Election Bloc,BYuT
+2231,Our Ukraine - People's Self-Defense,NU-NS
+2235,Agrarian Party of Russia,APR
+2236,Communist Party of the Russian Federation,KPRF
+2244,Just Russia,SR
+2245,Zhirinovsky Bloc,BZ
+2247,Our Home - Russia,NDR
+2252,Russian United Democratic Party 'Yabloko',Yabloko
+2253,Russia’s Democratic Choice,DVR
+2255,Union of Right Forces,SPS
+2257,Women of Russia,ZR
+2260,Party of Communists of the Republic of Moldova,PCRM
+2264,Christian Democratic People’s Party,PPCD
+2265,Democratic Party of Moldova,PDM
+2271,Liberal Democratic Party of Moldova,PLDM
+2272,Liberal Party,PL
+2280,Brothers of Italy,FDI
+2305,Democratic Party,DP
+2306,Liberty Forward Party,LFP
+2307,New Frontier Party,NFP
+2346,New Serbia,NS
+2347,Croatian Democratic Assembly of Slavonia and Baranja,HDSSB
+2348,Independent Democratic Serbian Party,SDSS
+2415,Autonomy Liberty Democracy Aosta Valley,ALD
+2447,New Social Democratic Party,NSDP
+2449,Liberal Party,LP
+2458,Together 2014 -Dialogue for Hungary Electoral Alliance,E14-PM
+2517,Bright Future,Bf
+2538,Commitment-Q,PRC
+2545,United Liberal Party,ULD
+2546,Democratic Labour Party,DLP
+2547,Democratic Party - including Platform Party,DP
+2548,Millenium Democratic Party,MDP
+2553,Party of Socialists of the Republic of Moldova,PSRM
+2645,Independent Social Alliance,ASI
+2874,Broad Front for Democracy,FAD
+2915,Yugoslav Left,JUL
+2988,Georgian Dream,GD
+3098,Modern Centre Party,SMC
+3104,Socialist People's Party of Montenegro,SNP CG
+3114,Party of Alenka Bratušek,SAB
+3128,There is a Future,YA
+3131,Meretz,MERETZ
+3143,Croatian Labourists - Labour Party,HL
+3162,Democratic Party of Socialists of Montenegro,DPS CG
+3163,For a Better Life,DZB
+3164,Liberal Alliance of Montenegro,LSCG
+3168,Croatian Democratic Union 1990,HDZ 1990
+3171,Alliance for a Better Future of Bosnia and Herzegovina,SBB BiH
+3173,Justice and Reconciliation Party,SPP
+3176,Party of Democratic Action,SDA
+3177,Serbian Progressive Party,SNS
+3185,Social Democratic Party of Montenegro,SDP CG
+3187,Alternative for Bulgarian Revival,ABV
+3194,Latvian Association of Regions,LRa
+3203,We Can,Podemos
+3210,People's Movement Party,PMP
+3217,Citizens - Party of the Citizens,C's
+3218,Voice,VOX
+3225,Free,L
+3229,Union of Democrats and Independents,UDI
+3252,New Serbian Democracy,NOVA
+3255,Bosniak Party,BS
+3266,All-Ukrainian Union 'Fatherland',Batkivshchyna
+3267,All-Ukrainian Union ‘Freedom',ВО/Свобода
+3271,Free Party,EVA
+3273,All of Us,Kulanu
+3645,People's Party,NS
+3671,New Space,NE
+3698,Red Party,R
+3783,Heritage,HePa
+3890,Armenia Alliance,HD
+3904,alliance: HDZ-HK~HNZ,HDZ-HK~HNZ
+3908,Gergiovden-VMRO / VMRO-Bulgarian National Movement,G-VMRO; VMRO-BND
+3915,Approve Dignity,FA
+3916,Colombian Coalition,CC
+3955,Labour - Gesher,Avoda-Gesher
+3973,Coalition Civic Option for Macedonia,GROM
+3979,Commitment for Mexico,AM
+3982,Democratic Coalition,DK
+3995,Opposition Action Alliance,AAO
+3997,Alliance Possible Peru,PP
+4011,We must,MORAMO
+4020,alliance: CP / VLSSP,CP / VLSSP
+4044,Proud and Sovereign Fatherland Alliance Movement,Alianza PAIS
+4070,Alternativ,A
+4094,Conservative People's Party of Estonia,EKRE
+4182,Front for Loyalty and Union of the Democratic Centre,FPL + UCeDé
+4205,Prosperous Armenia,BHK
+4214,People’s Force,FP
+4258,Internal Macedonian Revolutionary Organization-People's Party,VMRO-NP
+4269,Creating Opportunities,CREO
+4364,Liberal Democracy,DL
+4373,Justice Party,AP
+4400,Democratic and Popular Union,UDP
+4405,Liberal Party,PR
+4418,National Integration Party,PIN
+4546,Congress of the People,COPE
+4547,Democratic Party,DP
+4548,Economic Freedom Fighters,EEF
+4628,Republicans,Rep
+4630,Modern,.N
+4631,Kukiz'15,K
+4714,United Left,ZL
+4758,Democratic Party,DP
+4766,Movement for Changes,PzP
+4778,European Solidarity,BPP
+4779,Opposition Bloc,OB
+4795,Democratic Convergence of Catalonia,CDC
+4852,Democratic Alliance - 19th of April Movement,DA M19
+4865,Bridge of Independent Lists,MOST
+4866,People's Party - Reformists,NS-R
+4870,Social Democrats,DS
+4873,We Are Family,SR
+5414,Lanka Equal Society Party,LSS
+5453,Centre Alliance,NXT
+5454,Reform Party,Vidreisn
+5468,United Reform Action,URA
+5486,Serbian People's Party,SNS
+5553,Front for a Country in Solidarity,FREPASO
+5650,Europe Ecology - The Greens,EELV
+5848,Simeon II Coalition,KSII
+5852,People’s Party,FlF
+5855,Forum for Democracy,FvD
+5856,DENK,DENK
+5857,Renaissance,REM|R
+5858,Indomitable France,FI
+5879,Democratic Center,CD
+5926,Green Alliance,AV
+5939,Portugal Ahead,PàF
+5969,Save Romania Union,USR
+5976,Social Democratic Party 'Harmony',SDPS
+6042,Peoples' Democratic Party,HDP
+6087,Democratic Front,DF
+6110,National Regeneration Movement,MORENA
+6113,Social Encounter Party,PES
+6114,Alliance of Patriots of Georgia,APG
+6125,Freedom and Direct Democracy,SPD
+6131,The Left,L
+6135,Constitutional Democratic Party of Japan,CDP
+6150,Democratic Movement of Serbia,DEPOS
+6159,Civic Coalition,CC
+6560,Justice Party,JP
+6561,Democratic Party,DPK
+6648,Federal Peronism / Dissident Peronism,PF-PJ
+6691,Communist Party of Sri Lanka,CP
+6804,Lithuanian Christian Democratic Party,LKDP
+7049,Together for Change,JxC
+7339,The New Right,NB
+7348,European Realistic Disobedience Front,MeRA25
+7421,Together for Catalonia,JxCat
+7565,Centre Party,M
+7599,Blue and White,KL
+7619,National Alliance,NA
+7909,Movement for Change,KINAL
+7912,Joint List,JL
+8031,In Common We Can,ECP
+8054,Democrats of the Left,DS
+8058,Forza Italia,FI
+8168,The Republicans,LR
+8176,Geneva Citizens' Movement,MCG
+8182,Enough,CH
+8241,Solidarity–People Before Profit,PBPS
+8393,Development/For!,AP!
+8640,Liberal Iniciative,IL
+8842,Coalition of the Social Democratic Party of Croatia and the Croatian Social-liberal Party,SDP-HSLS
+9001,Ivica Dacic - Prime Minister of Serbia,SPS/JS/ZS
+9095,Rightwards,Y
+218,Polo Democratico Alternativo,PDA
+284,Unified Democratic Coalition,CDU
+568,Nationalist Revolutionary Movement,MNR
+751,SNK European Democrats,SNK-ED
+1036,Likud,Likud
+1060,Republican People's Party,CHP
+1308,Social Democratic Centre,CDS-PP
+1665,Coalition for Bulgaria,BSP
+1674,VMRO-DPMNE,VMRO-DPMNE
+1724,Radical Civic Union,UCR
+1985,Popular Socialist Party,PPS
+2234,Party of Regions,PR
+2256,United Russia,ER
+3677,Jewish Home,HaBayit
+4717,Democratic Front,DF
+7031,Left Ecology Freedom,SEL

data/union_mapping.csv

@@ -0,0 +1,107 @@
+manifesto_pf_id,manifesto_name,expert_pf_id,expert_name,country,relationship,status,detection_method
+3889,PJ,6648,PF-PJ,AR,Peronist faction no independent manifesto,implemented,manual
+6161,FAP,1365,PS,AR,"LLM: PS (Socialist Party) was a core constituent of FAP (Frente Amplio Progresista), which published joint manifestos.",implemented,llm_verified
+6161,FAP,6160,FR,AR,"LLM: FR (Frente Renovador) joined FAP in some elections, but also ran independently; likely constituent in FAP manifestos.",implemented,llm_verified
+6161,FAP,6554,FPCyS,AR,"LLM: FPCyS (Frente Progresista, Cívico y Social) included PS and others; manifestos often under FAP or similar coalitions.",implemented,llm_verified
+486,LPA,1998,LP,AU,"LLM: The Liberal Party (LP, PF ID: 1998) is the predecessor and constituent of the Liberal Party of Australia (LPA, PF ID: 486); manifestos are published under LPA, not LP.",implemented,llm_verified
+1743,NPA,338,NAT,AU,"LLM: The National Party (NAT) is the predecessor and constituent of the National Party of Australia (NPA, PF ID: 1743), which is the name used for joint manifestos after the party's rebranding; manifestos are published under NPA, not NAT.",implemented,llm_verified
+1760,HDZ BiH,3904,HDZ-HK~HNZ,BA,HDZ-led coalition variants,implemented,manual
+36,N-VA,756,CD+NVA,BE,cartel list,implemented,manual
+604,CD/V,622,CD&V,BE,same party duplicate PF ID,implemented,manual
+604,CD/V,756,CD+NVA,BE,"LLM: CD+NVA refers to the joint electoral lists of CD&V and N-VA (mainly 2003-2007); during this period, they published joint manifestos under the CD/V (PF ID: 604) label in the text data.",implemented,llm_verified
+1680,sp.a,1586,sp.a-SPIRIT,BE,merger,implemented,manual
+374,NDSV,5848,KSII,BG,"LLM: KSII is the coalition led by NDSV (374); manifestos published under NDSV, not KSII.",implemented,llm_verified
+482,SDS,3908,G-VMRO; VMRO-BND,BG,bloc constituent (progtype=8),implemented,progtype_8
+1765,ONS,3908,G-VMRO; VMRO-BND,BG,bloc constituent (progtype=8),implemented,progtype_8
+5649,Patriotic Front - NFSB and VMRO,2057,NFSB,BG,bloc constituent (progtype=8),implemented,progtype_8
+5649,Patriotic Front - NFSB and VMRO,3908,G-VMRO; VMRO-BND,BG,bloc constituent (progtype=8),implemented,progtype_8
+360,FDP/PLR,1231,FDP/PLR,CH,same party duplicate PF ID,implemented,manual
+6061,Alliance,928,RN,CL,core coalition member joint CMP manifesto,implemented,manual
+6061,Alliance,1599,UDI,CL,core coalition member joint CMP manifesto,implemented,manual
+1707,STAN,751,SNK-ED,CZ,"LLM: SNK-ED ran joint lists and published manifestos together with STAN (PF ID: 1707) in parliamentary elections, notably in 2006, under the SNK-ED–STAN label.",implemented,llm_verified
+2138,LB,1041,KSC,CZ,bloc constituent (progtype=8),implemented,progtype_8
+6202,KDU-ČSL-US-DEU,104,US-DEU,CZ,bloc constituent (progtype=8),implemented,progtype_8
+211,CDU/CSU,1375,CDU,DE,constituent,implemented,manual
+211,CDU/CSU,1731,CSU,DE,constituent,implemented,manual
+1816,B90/Grüne,10,Die Grünen,DE,merger,implemented,manual
+3925,RED-ID,797,ID,EC,constituent of alliance,implemented,manual
+685,RP,491,ERP,EE,LLM: ERP (Res Publica) published joint manifestos as 'RP' (PF ID: 685) in the text data; Res Publica is the Estonian name for the same party.,implemented,llm_verified
+779,I/ERSP,908,RKI,EE,bloc constituent (progtype=8),implemented,progtype_8
+779,I/ERSP,1299,ERSP,EE,merger,implemented,manual
+139,CiU,4795,CDC,ES,constituent,implemented,manual
+8271,Compromís–Podemos–EUPV,5623,CC,ES,LLM: CC (Compromís) published joint manifestos as part of Compromís–Podemos–EUPV (PF ID: 8271) in general elections.,implemented,llm_verified
+213,MoDem,496,MoDem,FR,same party duplicate PF ID,implemented,manual
+1108,EELV,5650,EELV,FR,same party duplicate PF ID,implemented,manual
+1595,UMP,4628,Les Républicains,FR,UMP renamed 2015,implemented,manual
+1595,UMP,8168,LR,FR,same as Les Républicains duplicate PF ID,implemented,manual
+1468,PASOK,7909,KINAL,GR,PASOK-dominated umbrella rebranded 2022,implemented,manual
+7347,EL,378,OP,GR,LLM: Oikologoi Prasinoi (OP) ran jointly with MeRA25 in 2019 as part of the 'MeRA25-Alliance for Breakup' and did not publish a separate manifesto; their program was subsumed under MeRA25.,implemented,llm_verified
+1475,SDP,8842,SDP-HSLS,HR,joint list SDP dominant,implemented,manual
+2522,Kukuriku,78,DC,HR,"LLM: DC (Democratic Centre) was a constituent of the Kukuriku coalition, which published joint manifestos under the Kukuriku name (PF ID: 2522) in 2011 and 2015.",implemented,llm_verified
+3648,ZL,8036,HKDU,HR,bloc constituent (progtype=5),implemented,progtype_5
+3918,DA-IDS-RDS,513,IDS,HR,constituent of alliance,implemented,manual
+242,PBP,8241,PBPS,IE,LLM: PBPS (Solidarity–People Before Profit) publishes joint manifestos under PBP (PF ID: 242) in the text data; this is a union.,implemented,llm_verified
+201,UdC,1758,UC,IT,LLM: Unione di Centro (UC/UDC) is a constituent of UdC (PF ID: 201) in the text data; they publish joint manifestos under UdC.,implemented,llm_verified
+1212,SEL,7031,SEL,IT,LLM: Sinistra Ecologia Libertà (SEL) is present in both datasets and publishes manifestos under SEL (PF ID: 1212) in text data.,implemented,llm_verified
+1737,Olive Tree,878,DS,IT,bloc constituent (progtype=8),implemented,progtype_8
+6241,House of Freedom,813,AN,IT,bloc constituent (progtype=8),implemented,progtype_8
+6241,House of Freedom,1519,CeD,IT,bloc constituent (progtype=8),implemented,progtype_8
+1967,SLFP,4020,CP / VLSSP,LK,"LLM: CP/VLSSP often contested as part of the SLFP-led United Front and People's Alliance, typically under joint manifestos with SLFP, but sometimes ran separately; medium confidence due to occasional independent runs.",implemented,llm_verified
+1967,SLFP,5414,LSS,LK,"LLM: LSSP was a core constituent of the SLFP-led United Front and People's Alliance, publishing joint manifestos with SLFP.",implemented,llm_verified
+1967,SLFP,6691,CP,LK,"LLM: CP was a constituent of the SLFP-led United Front and People's Alliance, publishing joint manifestos with SLFP.",implemented,llm_verified
+197,BSDK,168,LRS,LT,bloc constituent (progtype=8),implemented,progtype_8
+197,BSDK,1747,LMP-NDP,LT,bloc constituent (progtype=8),implemented,progtype_8
+377,LTS,1410,LLaS,LT,bloc constituent (progtype=8),implemented,progtype_8
+5779,SK,1407,LZP,LT,bloc constituent (progtype=8),implemented,progtype_8
+186,LSAP/POSL,898,SDP,LU,same party duplicate PF ID,implemented,manual
+708,LNNK-LZP,1296,LZP,LV,name fragment of LNNK-LZP,implemented,name_fragment
+1704,TB-LNNK,671,TB,LV,constituent of alliance,implemented,manual
+1704,TB-LNNK,1789,LNNK,LV,constituent of alliance,implemented,manual
+1704,TB-LNNK,7619,NATBLNNK,LV,"LLM: NATBLNNK is a constituent of TB-LNNK (1704), which published joint manifestos as a union.",implemented,llm_verified
+7622,ACUM,7904,PAS,MD,bloc constituent (progtype=8),implemented,progtype_8
+3254,DSCG,3253,HGI,ME,LLM: HGI (Croatian Civic Initiative) is a small Croatian minority party that typically runs on joint lists with DSCG (Democratic Union of Croats) in Montenegro; manifestos are published under DSCG.,implemented,llm_verified
+1537,GL,1533,Groen,NL,"LLM: Groen was a constituent party of GroenLinks (GL, PF ID: 1537), which published joint manifestos after its formation; Groen did not publish separate manifestos after joining the union.",implemented,llm_verified
+716,Alliance,1119,NLP,NZ,LLM: NLP (NewLabour Party) was a founding constituent of the Alliance and published joint manifestos under the Alliance name.,implemented,llm_verified
+4219,C90,5130,P2000,PE,"LLM: P2000 (Perú 2000) was a bloc including Cambio 90 (C90); manifestos were published under the C90/NM/P2000 bloc, which is represented as C90 (PF ID: 4219) in text data.",implemented,llm_verified
+1458,WAK,70,ZChN,PL,bloc constituent (progtype=8),implemented,progtype_8
+8268,UW,1566,D|W|U,PL,"LLM: D|W|U refers to Unia Wolności (UW) and its predecessors/successors, which published manifestos under the UW name (PF ID: 8268) in the text data.",implemented,llm_verified
+192,CDR,645,PAC,RO,bloc constituent (progtype=8),implemented,progtype_8
+1347,PSD-PUR,1443,PU|PC,RO,name fragment of PSD-PUR,implemented,name_fragment
+5941,USL,120,PSD,RO,PSD was the lead constituent of USL coalition (2012 joint manifesto),implemented,manual
+5941,USL,481,PNL,RO,PNL was a core constituent of USL coalition (2012 joint manifesto),implemented,manual
+5941,USL,1541,UNPR,RO,LLM: UNPR was a constituent of the USL (PF ID: 5941) coalition and did not publish its own manifesto; manifestos were issued under the USL name.,implemented,llm_verified
+6153,PSD-PC,1443,PU|PC,RO,name fragment of PSD-PC,implemented,name_fragment
+8626,LDP/LSV/SDS,4769,LSV,RS,name fragment of LDP/LSV/SDS,implemented,name_fragment
+205,SV,200,SDSS,SK,bloc constituent (progtype=8),implemented,progtype_8
+226,SDK,200,SDSS,SK,bloc constituent (progtype=8),implemented,progtype_8
+1617,SDKÚ-DS,983,DS,SK,DS merged into SDKÚ-DS,implemented,manual
+6629,DÚS,707,DUS,SK,LLM: DUS (Demokratická únia Slovenska) is the same as DÚS (PF ID: 6629) in the manifesto data; manifestos are published under the union name.,implemented,llm_verified
+1658,FA,3671,NE,UY,"LLM: NE (Nuevo Espacio) is a well-known constituent party of the Frente Amplio (FA) coalition, which publishes joint manifestos under the FA name.",implemented,llm_verified
+301,"SYRIZA, SYN; SYRIZA, Syriza, SYN/SYRIZA",1682,DIKKI,GR,"LLM (bloc-centric): DIKKI was a constituent member of the SYRIZA bloc in the 2007 election, running under its banner and publishing joint manifestos.",implemented,llm_verified
+676,"KDU, KDU-ČSL, KDU-CSL, KDU/CSL, KDUCSL, KDU–CSL, KDU–Č, KDU- ČSL, CSL",824,KDS,CZ,LLM (bloc-centric): KDS (Christian Democratic Party) is explicitly listed as a constituent member of the 'Christian and Democratic Union - Czech People's Party' bloc in the PartyFacts comments and published joint manifestos with it.,implemented,llm_verified
+701,"ZZS, LZS",1702,LZS,LV,"LLM (bloc-centric): Latvijas Zemnieku savienība (LZS) was a core constituent member of the Greens' and Farmers’ Union (ZZS) bloc in 2002, publishing joint manifestos and running under the bloc's banner.",implemented,llm_verified
+852,"V, Unity, UNITY, VIENOTIBA, JV, PS",1531,JL,LV,LLM (bloc-centric): Jaunais laiks (JL) was a founding constituent of the Unity (Vienotība) bloc in 2010 and published joint manifestos under its banner.,implemented,llm_verified
+2190,"DSS, DSS/NS",2346,NS,RS,"LLM (bloc-centric): New Serbia (NS) was a verified constituent member of the 'Democratic Party of Serbia/New Serbia' bloc in the 2008 elections, publishing joint manifestos with DSS.",implemented,llm_verified
+2530,"FpV, FPV, FPV-PJ, AFplV, FplV",623,PJ,AR,"LLM (bloc-centric): The Justicialist Party (PJ) was the principal and founding constituent of the Front for Victory (FpV) bloc, running under its banner and publishing joint manifestos in all relevant elections.",implemented,llm_verified
+3906,NA,356,PT,BR,"LLM (bloc-centric): PT (Partido dos Trabalhadores) was the leading party and consistent constituent of this left/progressive bloc across all listed elections, publishing joint manifestos under its banner.",implemented,llm_verified
+3906,NA,723,PSB,BR,"LLM (bloc-centric): PSB (Partido Socialista Brasileiro) was a frequent coalition partner and constituent member of this bloc, including joint manifestos in several elections (notably 2002, 2006, 2010, 2014, and 2022).",implemented,llm_verified
+3906,NA,1009,PDT,BR,"LLM (bloc-centric): PDT (Partido Democrático Trabalhista) was a constituent member of this bloc in multiple elections, including joint manifestos (notably 2010, 2018, and 2022).",implemented,llm_verified
+3906,NA,4405,"PR, PR (2), PR / PL, PR/PL, PR PL, PL/PR",BR,"LLM (bloc-centric): PR (Partido da República) was a constituent member of the bloc in the 2010 and 2014 elections, publishing joint manifestos with the bloc.",implemented,llm_verified
+3906,NA,458,PTB,BR,"LLM (bloc-centric): PTB (Partido Trabalhista Brasileiro) was a constituent member of the bloc in the 2002 and 2006 elections, participating in joint manifestos.",implemented,llm_verified
+3906,NA,1823,PL,BR,"LLM (bloc-centric): PL (Partido Liberal) was a constituent member of the bloc in the 2002 election, publishing a joint manifesto.",implemented,llm_verified
+4550,"C, Concertacion, CPD",6,PS,CL,"LLM (bloc-centric): The Socialist Party of Chile (PS) was a founding and continuous member of the Concertación/CPD, running on joint lists and publishing joint manifestos.",implemented,llm_verified
+4550,"C, Concertacion, CPD",54,PPD,CL,"LLM (bloc-centric): The Party for Democracy (PPD) was a core constituent of the Concertación/CPD, participating in all its joint electoral platforms and manifestos.",implemented,llm_verified
+4550,"C, Concertacion, CPD",390,PDC,CL,"LLM (bloc-centric): The Christian Democratic Party (PDC) was a principal founding member of the Concertación/CPD, running under its banner and signing joint manifestos.",implemented,llm_verified
+4550,"C, Concertacion, CPD",437,PRSD,CL,"LLM (bloc-centric): The Radical Social Democratic Party (PRSD) was a constituent member of the Concertación/CPD, participating in joint electoral lists and manifestos.",implemented,llm_verified
+8999,"ZMS, Aleksandar V..., PS-TN, Serbia is Wi..., ally",3177,SNS,RS,LLM (bloc-centric): The Serbian Progressive Party (SNS) was the leading and founding constituent of the 'Aleksandar Vučić – Serbia wins / For Our Children / Serbia Must Not Stop' bloc in all listed elections and published joint manifestos under this banner.,implemented,llm_verified
+1117,PO,4630,.N,PL,Nowoczesna was core constituent of Koalicja Obywatelska (KO) in 2019 under PO CMP code (progtype=8),implemented,manual
+4550,Concertacion,162,PC,CL,Communist Party of Chile was constituent of Nueva Mayoría (2013-2017) under Concertación PF ID,implemented,manual
+4550,Concertacion,209,PH,CL,Humanist Party was Concertación constituent (2005-2009),implemented,manual
+5668,EH Bildu,1671,Amaiur,ES,Amaiur (2011) predecessor to EH Bildu; expert data 2014-2024 covers EH Bildu years,implemented,manual
+6241,CdL,1767,CCD,IT,CdL coalition constituent (1994-2008),implemented,manual
+3979,Salvemos a México,1474,PRI,MX,PRI-PVEM electoral coalition (2006-2012) under various names,implemented,manual
+3979,Salvemos a México,446,PVEM,MX,PRI-PVEM electoral coalition (2006-2012) under various names,implemented,manual
+7912,Joint List,421,Hadash,IL,Arab party coalition (2015-2021); Hadash is core constituent,implemented,manual
+7912,Joint List,1663,Balad,IL,Arab party coalition (2015-2021); Balad is constituent,implemented,manual
+365,PdL,1626,FI,IT,merger constituent,implemented,manual
+365,PdL,813,AN,IT,merger constituent,implemented,manual

docs/DATA_CODING_PRINCIPLES.md

@@ -0,0 +1,240 @@
+# Data Coding Principles for 4D Latent Trait Model
+
+## V4 Implementation (Current Version)
+
+**As of V4 (2025-11-18), manifesto items implement the bipolar bridge structure described in this document.**
+
+**Key Changes from V3.x**:
+- ✅ Manifesto items now load on TWO dimensions (bipolar bridges)
+- ✅ Data format: `type_high` and `type_low` columns replace single `type`
+- ✅ Stan model: Unified `Gamma_man` matrix replaces per-dimension arrays
+- ✅ Measurement consistency: Manifesto items match expert data structure
+
+**Why V4?**
+- Better identification (each observation informs two dimensions)
+- Estimated correlations (not imposed by construction)
+- No double-counting (each quasi-sentence counted once)
+
+See CHANGELOG.md for full V4 migration details.
+
+---
+
+## Model Structure Overview
+
+The model estimates **four unipolar latent dimensions**:
+- **pro_market**: Support for market liberalization
+- **pro_welfare**: Support for welfare state expansion
+- **cosmopolitan**: Support for internationalism, diversity, openness
+- **traditional**: Support for nationalism, security, traditional values
+
+These are **separate dimensions**, not two bipolar scales. Correlations between dimensions (e.g., cosmopolitan-traditional) are **estimated empirically**, not imposed by construction.
+
+---
+
+## Item Types and Loading Structure
+
+### 1. Bipolar Bridge Items
+
+**Definition**: Items where the sample includes mentions of BOTH sides of an issue, and "positive" counts mentions favoring one pole.
+
+**Structure**:
+- `sample` = mentions of issue (any direction)
+- `positive` = mentions favoring one pole
+- `positive/sample` ratio varies from 0 to 1
+
+**Loading**: Should load on **ONE dimension only**
+
+**Examples**:
+
+**Manifesto Data**:
+```
+var: "Multiculturalism"
+type: "cosmopolitan"
+sample: per607 (pro-multiculturalism) + per608 (anti-multiculturalism)
+positive: per607 (pro-multiculturalism)
+```
+- High ratio → high cosmopolitan (party favors multiculturalism)
+- Low ratio → low cosmopolitan (party opposes multiculturalism)
+- Anti-multiculturalism is **implicitly measured** as (sample - positive)
+
+**PolDem Data**:
+```
+var: "Immigration (Media)"
+type: "cosmopolitan"
+sample: all immigration mentions (direction != 0)
+positive: pro-immigration mentions (direction > 0)
+```
+- High ratio → high cosmopolitan (media coverage shows party supporting immigration)
+- Low ratio → low cosmopolitan (media coverage shows party opposing immigration)
+
+### 2. Why One Loading Suffices for Bipolar Items
+
+**Question**: Shouldn't anti-immigration also load on traditional?
+
+**Answer**: No, because:
+
+1. **Both poles are already captured**: The bipolar structure means low cosmopolitan (anti-immigration) is automatically measured
+2. **Avoids double-counting**: Each mention/quasi-sentence contributes to exactly ONE item
+3. **Empirical correlations emerge naturally**: If anti-immigration parties also score high on nationalism/law-and-order, the **posterior correlation** between cosmopolitan and traditional will reflect this
+4. **More flexible model**: Cosmopolitan-traditional relationship is **estimated**, not imposed
+
+**Imposed vs. Estimated Correlation**:
+- If we double-load immigration on both cosmopolitan (negative) and traditional (positive), we **force** them to be opposites
+- By loading only on cosmopolitan, we let the data reveal whether anti-immigration parties are also nationalist (empirical question)
+
+---
+
+## Coding Decision Rules
+
+### Rule 1: Each Manifesto Code Appears in ONE Item Only
+
+**Good** (current structure):
+```
+"Multiculturalism" (cosmopolitan):
+  - per607 (Positive), per608 (Negative)
+
+"National Identity" (traditional):
+  - per601 (Positive), per107 (Negative)
+```
+- per607/per608 only in cosmopolitan
+- per601/per107 only in traditional
+- Correlation between dimensions is empirical
+
+**Bad** (double-loading):
+```
+"Multiculturalism" (cosmopolitan):
+  - per607 (Positive), per601 (Negative)
+
+"National Identity" (traditional):
+  - per601 (Positive), per607 (Negative)
+```
+- per601 and per607 counted twice
+- Imposes perfect negative correlation between cosmopolitan/traditional
+
+### Rule 2: Stance Assignment Within Items
+
+Within each item (var), codes are assigned stance based on:
+- **Positive**: Codes indicating support for the item's construct
+- **Negative**: Codes indicating opposition to the item's construct
+
+**Example - "Internationalism" (cosmopolitan)**:
+- per107 (Internationalism positive): stance = "Positive"
+- per109 (Internationalism negative): stance = "Negative"
+- Result: High per107 / low per109 → high cosmopolitan score
+
+### Rule 3: PolDem Direction Mapping
+
+PolDem uses `direction` variable (-1, 0, +1):
+- `direction > 0`: Support for the issue as coded
+- `direction < 0`: Opposition to the issue
+- `direction == 0`: Ambivalent (exclude from analysis)
+
+**Aggregation**:
+```r
+poldem_processed %>%
+  filter(direction != 0) %>%  # exclude neutral
+  group_by(party, year, country, issue_cat) %>%
+  summarise(
+    sample = n(),  # all non-neutral mentions
+    positive = sum(direction > 0)  # supportive mentions only
+  )
+```
+
+---
+
+## Special Cases
+
+### Immigration (Direction Ambiguity)
+
+**Codebook says**: "Opposition to restrictive immigration"
+
+**Interpretation needed**: Does `direction = +1` mean:
+- A) Support for "opposition to restrictions" → pro-immigration (cosmopolitan)
+- B) Support for "restrictions" → anti-immigration (traditional)
+
+**Resolution**: Must manually inspect sample sentences before finalizing coding.
+
+If interpretation A is correct:
+```r
+issue_cat == "immig" & direction > 0  → positive for cosmopolitan
+issue_cat == "immig" & direction < 0  → negative for cosmopolitan
+```
+
+If interpretation B is correct:
+```r
+issue_cat == "immig" & direction > 0  → negative for cosmopolitan
+issue_cat == "immig" & direction < 0  → positive for cosmopolitan
+# (REVERSED)
+```
+
+### Europe/Euro Items
+
+EU integration naturally maps to cosmopolitan-traditional dimension:
+
+**Manifesto Data**:
+- Add new items using per108 (EU integration positive) and per106 (EU integration negative)
+- Create separate vars: "EU Integration Support" (cosmopolitan), "Euroskepticism" (traditional)
+
+**PolDem Data**:
+```r
+"EU Integration Support (Media)" (cosmopolitan):
+  issue_cat = "europe" or "euro"
+  sample = all mentions
+  positive = direction > 0 (pro-EU)
+
+"Euroskepticism (Media)" (traditional):
+  issue_cat = "europe" or "euro"
+  sample = all mentions
+  positive = direction < 0 (anti-EU)
+```
+
+**Note**: Same sentences contribute to BOTH items, but counting opposite directions. This creates natural negative correlation between cosmopolitan/traditional.
+
+**Alternative approach** (cleaner, recommended): Load only on cosmopolitan:
+```r
+"EU Position (Media)" (cosmopolitan):
+  issue_cat = "europe" or "euro"
+  sample = all mentions
+  positive = direction > 0
+```
+This is sufficient if we treat EU as a bipolar cosmopolitan item.
+
+---
+
+## Data Structure Requirements
+
+### Manifesto Data Format (party-year-var level)
+
+Each row represents one item for one party-year:
+
+| party | country | year | var | type | sample | positive | project |
+|-------|---------|------|-----|------|--------|----------|---------|
+| 211 | DE | 2013 | Multiculturalism | cosmopolitan | 45 | 23 | Manifesto |
+| 211 | DE | 2013 | National Identity | traditional | 67 | 58 | Manifesto |
+| 211 | DE | 2013 | Economic Intervention | pro_welfare | 102 | 78 | Manifesto |
+
+- **var**: Item name (e.g., "Multiculturalism", "Economic Intervention")
+- **type**: Dimension it loads on (pro_market, pro_welfare, cosmopolitan, traditional)
+- **sample**: Total quasi-sentences mentioning this issue
+- **positive**: Quasi-sentences with positive stance toward this item
+
+### PolDem Data Format (same structure)
+
+| party | country | year | var | type | sample | positive | project |
+|-------|---------|------|-----|------|--------|----------|---------|
+| 211 | DE | 2013 | Immigration (Media) | cosmopolitan | 23 | 8 | PolDem |
+| 211 | DE | 2013 | Nationalism (Media) | traditional | 15 | 12 | PolDem |
+
+Combined using `bind_rows()` to create unified dataset.
+
+---
+
+## Summary
+
+1. **Bipolar items load on one dimension only** - the ratio captures both poles
+2. **Each manifesto code appears in exactly one item** - no double-counting
+3. **Correlations between dimensions are estimated, not imposed** - more flexible model
+4. **Direction reversals are handled within items** - via stance assignment (Manifesto) or direction sign (PolDem)
+5. **All items must allow varying positive/sample ratios** - mix of positive and negative stances required
+
+This structure preserves the conceptual independence of the four dimensions while allowing the data to reveal their empirical relationships.

docs/FULL_RUN_OPERATIONS.md

@@ -0,0 +1,73 @@
+# Full model run operations
+
+This note describes how to launch, monitor, and summarize the long full model
+run that generates the posterior estimates.
+
+Do not start a full run until raw-data and Stan-data preflight checks pass.
+
+## Hardware note for runtime reporting
+
+The production run should record elapsed time for this workstation as:
+
+```text
+Hardware: 4 cores of an AMD Ryzen 9 7945HX
+```
+
+After the run, record the wall-time reported by `src/sh/show_run_progress.sh` or
+`outputs/model_outputs/latest/run_*/diagnostics/run_metrics.json`.
+
+## Launch with durable logging
+
+From the repository root:
+
+```bash
+mkdir -p outputs/logs
+ts="$(date +%Y%m%d_%H%M%S)"
+STAN_REFRESH=100 \
+  nohup bash run_estimation.sh full \
+  > "outputs/logs/full_run_${ts}.log" 2>&1 &
+echo $! > "outputs/logs/full_run_${ts}.pid"
+```
+
+The wrapper configures local raw data, local R libraries, project-local temp
+space, and the writable project-local CmdStan copy by default.
+
+## Monitor during the run
+
+Live log tail:
+
+```bash
+tail -f outputs/logs/full_run_<timestamp>.log
+```
+
+Convenience progress command:
+
+```bash
+bash src/sh/show_run_progress.sh
+```
+
+Stan progress is printed every `STAN_REFRESH` iterations (default `100`) and is
+captured in the durable log.
+
+## Inspect after completion
+
+After a successful run, the chain CSVs are under:
+
+```text
+outputs/model_outputs/latest/run_<timestamp>/chains/
+```
+
+Run diagnostics and Stan logs are archived under:
+
+```text
+outputs/model_outputs/latest/run_<timestamp>/diagnostics/
+```
+
+Use:
+
+```bash
+bash src/sh/show_run_progress.sh
+```
+
+to print the latest run status, wall time, sampler configuration, divergences,
+tree-depth hits, and command-configuration verification status.

docs/RAW_DATA_SOURCES.md

@@ -0,0 +1,67 @@
+# Raw data sources and local-only setup
+
+The public replication repository includes processed inputs under `data/`, but it
+does not redistribute licensed or third-party raw source files. Raw files needed
+to regenerate processed inputs should be kept in a local-only directory outside
+the replication git repository.
+
+Recommended local layout:
+
+```text
+_local/raw/
+  poldem/poldem-election_all.csv
+  manifesto/MPDataset_MPDS2025a.csv
+```
+
+The scripts read `PARTY2D_RAW_DATA_DIR` when it is set. For another location, use:
+
+```bash
+export PARTY2D_RAW_DATA_DIR=/path/to/local/raw
+```
+
+## Required raw inputs
+
+| Source | Raw file | Local path below `$PARTY2D_RAW_DATA_DIR` | Used by | Redistribution |
+| --- | --- | --- | --- | --- |
+| PolDem Election Campaigns, all issues | `poldem-election_all.csv` | `poldem/poldem-election_all.csv` | `src/r/00c_process_poldem.R` | Not redistributed in this repo |
+| Manifesto Project Dataset | `MPDataset_MPDS2025a.csv` | `manifesto/MPDataset_MPDS2025a.csv` | `src/r/00a_process_manifesto.R` if `data/manifesto_data.csv` is regenerated | Not redistributed in this repo |
+
+## PolDem download
+
+Dataset: `poldem-election_all` from the PolDem Election Campaigns collection.
+
+- Overview: <https://poldem.eui.eu/data-overview/>
+- Download page: <https://poldem.eui.eu/download/election-campaigns/>
+- CSV URL used locally on 2026-06-11:
+  <https://poldem.eui.eu/downloads/cosa/poldem-election_all.csv>
+
+Observed checksum after download on 2026-06-11:
+
+```text
+sha256  2cd8c9108b1b0b9c1b6594bb21acee709c70259cd02f450bc69fc09b505fc9fb
+```
+
+## Preflight check
+
+From the repository root, run:
+
+```bash
+bash src/sh/check_raw_data.sh
+```
+
+This checks required local raw inputs and prints byte sizes and SHA-256 checksums.
+
+## Processed files kept in this repository
+
+The processed files under `data/` are intended to be part of this repository,
+including:
+
+- `data/poldem_data.csv`
+- `data/manifesto_data.csv`
+- `data/text_data.csv`
+- `data/expert.csv`
+- `data/lr_data.csv`
+- `data/model_data.csv`
+
+These files document the analysis-ready inputs while avoiding redistribution of
+the underlying raw source data.

docs/union_mapping.md

@@ -0,0 +1,357 @@
+# Party Union Mapping
+
+## Individual Party Guarantee
+
+**Every `party_id` in the output CSV represents an individual political party, never an electoral alliance or bloc.**
+
+Electoral alliances and blocs are handled in one of two ways:
+1. **Decomposed via mean-constituent averaging** (N=123 mappings): Shared manifesto data feeds into individual constituent party estimates. The output contains the constituents, not the alliance.
+2. **Excluded with documented justification** (see "Excluded Alliance Labels" below): Alliance labels with no mappable constituents are dropped from the output.
+
+A systematic audit of all output parties is provided by `scripts/audit_party_types.py`, which produces `scripts/party_type_audit.csv` classifying every party with evidence. Post-estimation verification in `02_post_estimation.jl` hard-fails if any union/alliance PF ID appears in the output.
+
+## Overview
+
+Expert surveys often rate individual constituent parties separately, while manifesto data is published under a union/coalition/merged party name. This document maps individual party PartyFacts IDs to the union PartyFacts IDs used in `text_data.csv`.
+
+**Direction:** Individual party (expert data) → Union (manifesto/text data)
+
+## Model Integration (V4 Mean-Constituent Model)
+
+The V4 Stan model (`stan_model_2dim_v4.stan`) uses these mappings to produce **individual party estimates** for all constituent parties. Instead of collapsing expert data onto the union ID, V4 gives every constituent its own latent position (theta) and links shared manifesto data to the **mean** of constituent thetas.
+
+### How it works
+
+1. **Each constituent gets its own segment and random walk.** CDU (1375) and CSU (1731) each have independent theta trajectories. The union ID (211) gets NO segment.
+
+2. **Manifesto observations constrain the mean.** A CDU/CSU manifesto in year t enters the likelihood as:
+   ```
+   pos = mean(theta[dim, rr_CDU_t], theta[dim, rr_CSU_t])
+   ```
+   The observation is counted once (no double-counting), but it pulls both constituents' thetas via their average.
+
+3. **Expert data constrains individuals directly.** A CHES rating of CDU in 2019 maps to `theta[dim, rr_CDU_2019]` with no averaging. This is what identifies the *difference* between constituents.
+
+4. **Identification depends on data availability:**
+   - **Periods with individual expert data** (e.g., CHES 1999-2024): Constituent estimates separate meaningfully. CSU appears more conservative on galtan than CDU, matching known ground truth.
+   - **Periods without individual expert data** (e.g., 1950s-1990s): Only the shared manifesto constrains the mean. The random walk prior pulls constituents toward similar values, so CDU ≈ CSU with wide credible intervals on the gap. The estimates gradually differentiate as expert data appears.
+
+5. **Backwards compatible.** With an empty `union_mapping.csv`, all observations have `n_const=1` and V4 reduces exactly to V3.
+
+### Data flow
+
+```
+R pipeline (00_data-management.R):
+  - text_data keeps union PF IDs (211) for manifesto rows
+  - expert/lr_data keeps individual PF IDs (1375, 1731)
+  - Expert filtering: party in text_data OR party in constituent_parties
+
+Julia pipeline:
+  02_data_loading.jl  → loads union_mapping.csv → builds union_to_constituents dict
+  03_data_preparation.jl → creates segments for CONSTITUENTS (not unions)
+                         → builds flat arrays: n_const_man[], const_offset_man[], const_rr_man[]
+  04_model_execution.jl  → passes constituent arrays to Stan
+
+Stan model (stan_model_2dim_v4.stan):
+  - Manifesto likelihood: averages theta over constituents per observation
+  - Expert likelihoods: same averaging (nc=1 for individual obs = direct lookup)
+  - No new parameters vs V3 — only data block and likelihood computation change
+```
+
+### Output
+
+Post-estimation produces individual party rows (CDU=1375 and CSU=1731 as separate rows) with a `union_party_id` column (= 211 for both, NA for standalone parties). The anchor party is CDU (1375) instead of CDU/CSU (211).
+
+### Scale impact (smoke test, post-1990 data)
+
+- 471 parties → 483 segments, 461 with valid segments (vs ~449 without unions)
+- 708 union manifesto observations with multi-constituent averaging (3.2% of 21,991 total)
+- 23,427 total constituent entries in flat arrays
+- 9,870 segment-year positions (R)
+- No new model parameters (theta simply covers more segment-years)
+- Negligible performance impact: `nc=1` fast path for >96% of observations
+
+### Data quality (verified 2026-02-08)
+
+- **No double-counting**: No constituents appear in text_data alongside their union
+- **Union IDs verified absent from output** via post-estimation check in `02_post_estimation.jl`
+- **No chain mappings**: No PF ID serves as both union target and constituent of another union
+- **No duplicate rows** in text_data after deduplication
+- **All flagged-for-review parties** in audit script are confirmed individual parties (false positives from name patterns)
+
+## Detection Methodology
+
+`scripts/diagnose_party_mismatches.R` uses a multi-signal approach, ranked by reliability:
+
+### Phase 1: MARPOR `progtype` variable (definitive)
+
+The raw Manifesto Project data (`MPDataset_MPDS2025a.csv`) contains a `progtype` variable classifying each manifesto entry:
+
+| progtype | Meaning | Relevance |
+|----------|---------|-----------|
+| 1 | Party's own manifesto | Individual party, no union issue |
+| **2** | **Programme of 2+ parties (individual tracking)** | Joint manifesto: each constituent gets its own CMP code with identical scores |
+| 3 | Electoral manifesto by a single party | Individual party |
+| **4** | **Estimated from another party's programme** | Party had no manifesto, inherited another party's scores |
+| **5** | **Average of member parties' manifestos** | MARPOR computed average from constituent parties |
+| 6 | Other | Miscellaneous |
+| **8** | **Party bloc programme** | Bloc-level CMP code representing multiple parties |
+| 9 | Non-standard text | Various sources |
+
+**Phase 1a (progtype=2):** Group by `(country, date)` where progtype=2 to find which CMP codes shared a joint manifesto at each election. Map each to PF IDs. If some are in text_data and others are expert-only, create constituent mappings.
+
+**Phase 1b (progtype=5):** Average-of-members entries have their own CMP code. Find constituents via PartyFacts "composed of:" comments and manifesto party names.
+
+**Phase 1c (progtype=8):** Bloc-level entries have their own CMP code. Find constituents via PartyFacts "composed of:" comments and manifesto party names.
+
+**Phase 1d (progtype=4):** Proxy entries where MARPOR estimated from another party. Currently detects 0 expert-only parties with progtype=4.
+
+### Phase 2: PartyFacts metadata enrichment
+
+**Phase 2a (comment parsing):** Search text_data party comments for "composed of:" patterns. Match mentioned abbreviations against expert-only parties using word-boundary matching.
+
+**Phase 2b (parlgov "+" notation):** parlgov uses "+" to denote unions (e.g., "CDU+CSU", "CCD+CDU"). Parse fragments and match against expert-only parties.
+
+**Phase 2c (name fragment matching):** For text_data parties with "/" or "-" in names, split into fragments and match against expert-only parties. **Bilingual disambiguation:** if all fragments resolve to the same PF ID, the "/" separates language variants (e.g., Swiss SPS/PSS), not constituents.
+
+### Phase 3: LLM verification (optional)
+
+For remaining unmatched expert-only parties, uses gpt-4.1 via GESIS OpenWebUI with enriched prompts containing: all name_short values across datasets, year ranges, PartyFacts comments, progtype history, and the full list of text_data parties in the same country. Skip with `--skip-llm` flag.
+
+### Matching safeguards
+
+- **Word-boundary matching:** Uses `\b` regex boundaries to prevent substring false positives (e.g., "DS" matching "NDSI")
+- **Minimum 2-character terms:** Single-letter abbreviations (e.g., "K", "G") are excluded from matching
+- **Deduplication:** When the same mapping is detected by multiple methods, the highest-reliability method is kept (manual > progtype > comments > parlgov > name_fragment > LLM)
+- **Idempotent:** The script strips `detection_method` from existing mappings on re-run
+
+## How This Was Built
+
+1. **Original 36 mappings** (2026-02-06/07): String matching + LLM + RA verification + independent research
+2. **Expanded with progtype** (2026-02-07): Rewrote `scripts/diagnose_party_mismatches.R` to use MARPOR progtype (types 2, 4, 5, 8), PartyFacts comments, parlgov "+", and bilingual-aware name matching
+3. **LLM verification with gpt-4.1** (2026-02-07): Ran enriched prompts through gpt-4.1 via GESIS OpenWebUI for 345 remaining expert-only parties. Found 32 additional verified constituent mappings across 25 countries.
+4. **Bloc-centric LLM sweep** (2026-02-07): For each of 38 unmapped progtype=8 bloc parties in text_data, sent targeted gpt-4.1 queries asking which expert-only parties are actual constituents. Found 17 new mappings (Brazil PT-led coalitions, Chilean Concertación, Serbian blocs, Latvian unions, etc.).
+5. **Manual research** (2026-02-07): Verified remaining unmapped blocs. Added 4 manual mappings (PL: .N→KO; CL: PC→Concertación, PH→Concertación; ES: Amaiur→EH Bildu). Confirmed remaining 27 blocs have no mappable expert-only constituents (constituents either already in text_data, not in expert surveys, or in fluid coalitions).
+6. **Temporary election coalitions** (2026-02-08): Systematic analysis of progtype=2 joint manifesto groups. Grouped by identical CMP content (all 142 per* columns) to separate left and right coalitions within the same election. Found 9 unmapped expert-only parties that shared manifestos with text_data parties (Italy 2001/2006/2013, France 2017). Added CCD→CdL from orphan analysis (1 manual). Total: 10 new mappings.
+7. **Individual party guarantee cleanup** (2026-02-08): Removed 11 progtype_2/progtype_2_joint entries that mapped real individual parties as unions of zero-data coalition partners. Removed 2 chain mappings (union→union→constituent). Added 4 new mappings for genuine alliances (MX: Salvemos a México→{PRI, PVEM}; IL: Joint List→{Hadash, Balad}). Documented classification decisions for all 15 dual-progtype parties and 4 pure bloc labels. Added post-estimation verification check and audit script.
+
+## Mapping Table
+
+The canonical mapping is in `data/union_mapping.csv` with columns:
+- `manifesto_pf_id` — PartyFacts ID of union (in text_data)
+- `manifesto_name` — Union display name
+- `expert_pf_id` — PartyFacts ID of individual constituent (in expert_raw)
+- `expert_name` — Individual party display name
+- `country` — ISO2 country code
+- `relationship` — Description of the relationship
+- `status` — `implemented` (active in pipeline) or `pending` (awaiting implementation)
+- `detection_method` — How detected: `manual`, `progtype_2`, `progtype_2_joint`, `progtype_5`, `progtype_8`, `progtype_4`, `composed_comment`, `parlgov_plus`, `name_fragment`, `llm_verified`
+
+## Detection Method Breakdown
+
+| Method | Count | Description |
+|--------|-------|-------------|
+| llm_verified | 48 | gpt-4.1 verified (party-centric + bloc-centric) |
+| manual | 45 | Hand-verified mappings (includes MX/IL bloc mappings, CCD→CdL) |
+| progtype_8 | 23 | Bloc-level CMP entries matched to expert-only constituents |
+| name_fragment | 4 | "/" or "-" name splitting matched to expert-only parties |
+| composed_comment | 2 | PartyFacts "composed of:" comments |
+| progtype_5 | 1 | Average-of-members entries (Croatia ZL) |
+
+**Total: 123 rows** mapping 116 unique expert parties to 87 union parties across 41 countries
+
+## Bloc Coverage
+
+Of 49 progtype=8 bloc parties in text_data, 22 (44.9%) have at least one constituent mapped. The remaining 27 have no mappable expert-only constituents because:
+- Actual constituents already have their own text_data entries (e.g., KDNP, Yesh Atid, TB-LNNK)
+- Actual constituents are not in expert surveys (e.g., Hadash, Ra'am, Balad, VS, DKP)
+- Coalition membership is too fluid for static mapping (Panama, Colombia, Israel shifting coalitions)
+- The entry is a single party coded as a bloc (e.g., German Minority, Red-Green Unity List)
+
+## Temporary Election Coalitions (progtype=2) — REMOVED
+
+The progtype=2 joint manifesto mappings (Italy 2001/2006/2013, France 2017, Belgium 1971) were removed in the 2026-02-08 cleanup because they mapped real individual parties as "unions" of tiny coalition partners with zero data. See "Removed Mappings" below for details.
+
+The CCD(1767)→CdL(6241) mapping from orphan analysis remains, as CCD was a genuine long-term CdL constituent (progtype=1 in 1996, part of CdL bloc from 2001).
+
+## Unmappable Expert-Only Parties
+
+These parties appear in expert surveys but have no manifesto union to map to. Verified against `text_data_unfiltered.csv` (pre-temporal-filter, 1238 parties) on 2026-02-07.
+
+**Filter retest results:** 54 of the 80 original RA task parties have manifesto data in the unfiltered text_data, but all were dropped by the temporal continuity filter (requires ≥3 years with gaps ≤6). These parties have their own CMP data but too few observations. They are correctly handled via union mapping (where applicable) or excluded (where standalone). See `scripts/party_mismatch_ra_task.csv` for full year-level detail.
+
+| Expert PF ID | Name | Country | Manifesto years (unfiltered) | Expert years | Why unmappable |
+|-------------|------|---------|------------------------------|--------------|----------------|
+| 5623 | Compromís | ES | 0 (CMP codes map to different PF IDs) | 2 (2018, 2023) | No manifesto data under PF ID 5623. CMP codes 33098/33093/33914 are separate PartyFacts entries. |
+| 4363 | FDG (Front de Gauche) | FR | 1 (2012) | 1 (2014) | 1 CMP year only. PCF (1251) already has its own expert data. |
+| 5731 | NNP | ZA | 0 | 1 (1999) | Apartheid party that dissolved into ANC 2005. Ideologically incompatible mapping. |
+| 5553 | FREPASO | AR | 1 (1995) | 4 (1995-2001) | 1 CMP year only. Independent party; Alianza (1999) was separate CMP entity. |
+| 8122 | CF (Consenso Federal) | AR | 1 (2019) | 2 (2019-2020) | 1 CMP year only. One-off coalition. |
+| 4182 | FPL+UCeDé | AR | 1 (2003) | 3 (1987-1991) | 1 CMP year only. Incoherent entity (UCeDé 1987 ≠ FPL coalition 2003). |
+| 6160 | FR (Frente Renovador) | AR | 0 | 4 (2013-2019) | No CMP code at all. Major ideological shift 2013→2019. |
+| 5879 | CD (Centro Democrático) | CO | 1 (2014) | 3 (2014-2019) | 1 CMP year only. Uribe's party, independent. |
+| 4411 | PNI/PPN | CR | 0 | 1 (1974) | No CMP code. Defunct 1970s party. |
+| 7412 | EK/DEK | CY | 0 | 1 (1970) | No CMP code. Defunct 1970 far-right party. |
+| 3935 | United Opposition | GE | 1 (2008) | 1 (2008) | 1 CMP year only. Was anti-UNM alliance (not led by UNM). |
+
+## Rejected Mappings
+
+These were proposed by the RA but found incorrect during independent research:
+
+| Individual PF ID | Name | Proposed target | Why wrong |
+|-----------------|------|-----------------|-----------|
+| 5623 | CC/Compromís | 81 (CCa-PNC-NC) | Name coincidence. Compromís is a Valencian left party; CCa is a Canarian right party. |
+| 4363 | FDG | 1251 (PCF) | FDG was a multi-party alliance (PCF + Parti de Gauche). PCF already has its own 2014 expert data. Mapping FDG→PCF would double-count. |
+| 5731 | NNP | 1219 (ANC) | The NNP (ex-apartheid National Party) dissolved into ANC in 2005 as political capitulation. NNP positions are diametrically opposed to ANC on every dimension. |
+
+## Completeness Audit (2026-02-08)
+
+Systematic analysis of 631 orphan expert parties (with 4+ observations) across 46 countries (excluding FPTP systems like UK, US, Canada where party unions are not a meaningful concept). Cross-referenced with MARPOR progtype data and PartyFacts metadata.
+
+### Methodology
+
+1. **Progtype=2 sweep**: Identified all 93 MARPOR entries with progtype=2. Grouped into 27 content-identical subgroups. Found 12 expert-only parties with progtype=2 data; 10 were mappable (mapped above), 2 had no matching text_data party in their coalition.
+
+2. **Orphan analysis**: For each of 46 countries, identified expert-only parties not in text_data and not already mapped. Examined PartyFacts metadata (names, comments, ideology tags) and MARPOR progtype history for each.
+
+3. **Coalition verification**: For Italian orphans specifically, checked each party's MARPOR entries for progtype=2/8 membership and ideological alignment with existing text_data coalition parties.
+
+### Major standalone orphan parties (not coalition members)
+
+These are significant parties with substantial expert data but no manifesto/union data. They are genuinely standalone — not missed union constituents.
+
+| Country | Party | PF ID | Expert obs | Why standalone |
+|---------|-------|-------|-----------|----------------|
+| AT | BZÖ | 599 | 7 | Splinter from FPÖ; own CMP data filtered out |
+| BE | PTB/PVDA | 1753 | 14 | Independent far-left; never part of any coalition |
+| CZ | ANO | 2141 | 13 | Babiš party; own CMP data exists but too recent |
+| CZ | Piráti | 2047 | 10 | Standalone; own CMP data exists |
+| FR | REM/R | 5857 | 10 | Macron's party; too new for sufficient CMP data |
+| FR | FI | 5858 | 8 | Mélenchon's party; standalone |
+| IT | M5S | 2046 | 13 | Five Star Movement; progtype=1 only, standalone |
+| IT | FDI | 2280 | 10 | Fratelli d'Italia; progtype=1 only, standalone |
+| RO | USD | 120 | 23 | Social democratic bloc; own CMP code exists |
+| SK | OĽaNO | 2130 | 13 | Populist party; standalone |
+| CO | PCC | 1577 | 19 | Conservative party; own CMP code exists but filtered |
+| BR | PSDB | 225 | 13 | Social democrats; standalone (not PT coalition) |
+
+### Inactive union mappings
+
+These union targets exist in `union_mapping.csv` but are NOT in text_data, making their constituent mappings inactive:
+
+| Union PF ID | Name | Country | Constituents | Why inactive |
+|-------------|------|---------|-------------|--------------|
+| 1212 | SEL | IT | SEL(7031) | 1212 not in text_data |
+| 1737 | Olive Tree | IT | PCI(34), DS(878) | 1737 not in text_data; only 2 MARPOR years |
+
+Note: 962 (CCD+CDU→CDU-Italy) was removed from the mapping entirely because it created a chain (962 is both a constituent of UdC and a union target). See "Removed Mappings" below.
+
+### Conclusion
+
+The 123 mappings comprehensively cover: (1) all permanent unions with separate expert data, (2) all progtype=8 bloc parties with mappable expert-only constituents, and (3) CCD as an additional Italian coalition member identified through orphan analysis. Remaining orphan expert parties are genuinely standalone parties whose manifesto data was either filtered out by temporal continuity requirements or does not exist.
+
+## Removed Mappings (2026-02-08)
+
+### 11 progtype=2 joint manifesto entries removed
+
+These entries mapped real individual parties (with substantial text and expert data) as "unions" of temporary coalition partners that had zero text data AND zero expert data. Every constituent had no information to contribute, making the mapping harmful (it reduced real parties to averages with phantom partners).
+
+| Union PF ID | Union Name | Constituent PF ID | Constituent | Country | Why removed |
+|---|---|---|---|---|---|
+| 8054 | DS | 279 | M-DL | IT | M-DL has 0 text, 0 expert data; DS has 120 text, 3 expert |
+| 1404 | PRC | 1635 | PdCI | IT | PdCI has 0 text, 0 expert data; PRC has 58 text, 15 expert |
+| 1404 | PRC | 1711 | RnP | IT | RnP has 0 text, 0 expert data |
+| 6241 | CdL | 888 | NPSI | IT | NPSI has 0 text, 0 expert data; CdL's progtype_8 mappings (AN, CeD, FI) remain |
+| 6241 | CdL | 2415 | ALD | IT | ALD has 0 text, 0 expert data |
+| 768 | IdV | 115 | P-UDEUR | IT | P-UDEUR has 0 text, 0 expert data; IdV has 26 text, 8 expert |
+| 768 | IdV | 1369 | SVP | IT | SVP has 0 text, 0 expert data (under this PF ID) |
+| 1221 | Lega | 365 | PdL | IT | PdL has 0 text, 0 expert data; Lega has 77 text, 24 expert |
+| 1595 | UMP | 3229 | UDI | FR | UDI has 0 text, 0 expert data; UMP has 66 text, 18 expert |
+| 49 | openVLD | 622 | CD&V | BE | CD&V is a large party (own PF ID); mapping created false dependency |
+| 554 | PRL | 622 | CD&V | BE | Same issue: CD&V already has its own data pipeline |
+
+### 2 chain mapping entries removed
+
+These created chain dependencies (union A → union B → constituents), which the pipeline does not support:
+
+| Union PF ID | Union Name | Constituent PF ID | Constituent | Country | Why removed |
+|---|---|---|---|---|---|
+| 962 | CCD+CDU | 763 | CDU (Italy) | IT | 962 is itself a constituent of UdC (201). CDU-Italy (763) has 0 data. Chain: 201→962→763. |
+| 5939 | PàF | 1742 | AD | PT | 5939 is itself a constituent of CDS-PP (1308). AD (1742) has 0 data. Chain: 1308→5939→1742. |
+
+## New Mappings Added (2026-02-08)
+
+| Union PF ID | Union Name | Constituent PF ID | Constituent | Country | Rationale |
+|---|---|---|---|---|---|
+| 3979 | Salvemos a México | 1474 | PRI | MX | PRI-PVEM electoral coalition (2006-2012); PRI has extensive text + expert data |
+| 3979 | Salvemos a México | 446 | PVEM | MX | PVEM is second constituent; has its own text + expert data |
+| 7912 | Joint List | 421 | Hadash | IL | Arab party coalition (2015-2021); Hadash has CHES 2022 data |
+| 7912 | Joint List | 1663 | Balad | IL | Balad is constituent; has CHES 2022 data |
+
+## Excluded Alliance Labels
+
+These party IDs appear in text_data as bloc/alliance labels but are NOT decomposed via union mapping because no constituent has data in the pipeline:
+
+| PF ID | Name | Country | Text data | Expert data | Why excluded |
+|---|---|---|---|---|---|
+| 3995 | Alianza Acción Opositora | PA | 48 obs | 0 | No expert survey coverage for Panama. No constituents identifiable in pipeline. |
+
+Note: Several other progtype=8 bloc parties in text_data also have no mapped constituents (see "Bloc Coverage" below), but they remain in the output either because (a) they have their own expert data (e.g., 2988 Georgian Dream, 3916 Alianza Grande) or (b) they function as individual parties despite bloc coding (e.g., 1527 Enhedslisten, 1439 German Minority).
+
+## Classification Decisions
+
+### Dual-progtype parties (both progtype=1 and progtype=8)
+
+These 15 parties have MARPOR entries under both individual (progtype=1/3) and bloc (progtype=8) codes. Each was individually reviewed.
+
+**Classified as individual parties (no action needed):**
+
+| PF ID | Name | Country | Evidence |
+|---|---|---|---|
+| 57 | SLD | PL | Dominant Polish left party; bloc coding reflects coalition leadership, not alliance status |
+| 81 | CCa | ES | Canarian regionalist party; single party with local coalition leadership |
+| 1056 | SC | LV | Latvian party; dual coding reflects different election formats |
+| 1150 | SDE | EE | Estonian Social Democrats; individual party |
+| 1396 | Samfylkingin | IS | Icelandic Social Democratic Alliance; merged into single party |
+| 1439 | MN | PL | German Minority in Poland; single ethnic party coded as bloc |
+| 1527 | Enhedslisten | DK | Red-Green Alliance; functions as single party since 1989 |
+| 1691 | FiDeSz-KDNP | HU | FiDeSz dominant; KDNP (1412) has separate PF ID and data |
+| 2172 | ENM | GE | United National Movement; single party with bloc-era coding |
+| 2228 | BYuT | UA | Tymoshenko bloc; functions as single Ukrainian party |
+| 2252 | Yabloko | RU | Russian liberal party; individual entity |
+
+**Classified as individual parties after research:**
+
+| PF ID | Name | Country | Decision | Evidence |
+|---|---|---|---|---|
+| 506 | VL-TB/LNNK | LV | Individual (merger party) | National Alliance formed 2010 by merger of VL and TB/LNNK. Post-merger, functions as single party. MARPOR data 2010-2022. TB/LNNK (1704) has separate pre-merger data (1998-2014). Not mapped as union because 1704 is already a union target with its own constituents; mapping would create a chain. |
+| 1586 | sp.a-SPIRIT | BE | Already handled | Already mapped as constituent of sp.a (1680) in union_mapping.csv. 0 text, 0 expert data under this PF ID. |
+| 7599 | Kahol Lavan | IL | Individual party | Short-lived centrist party (2019-2020). Has own expert data (CHES 2021, V-Party 2019). Unified entity, not a multi-party alliance. |
+| 2988 | Georgian Dream | GE | Individual party | Despite progtype=8 coding, functions as a single party-movement. Has own expert data (GPS 2019, V-Party 2012/2016). |
+| 3916 | Alianza Grande | CO | Individual party (catch-all PF ID) | PF ID covers multiple Colombian coalitions. Has own expert data (CHES 2020). No separate constituent expert data exists. |
+
+**Classified as alliance and mapped:**
+
+| PF ID | Name | Country | Decision | Evidence |
+|---|---|---|---|---|
+| 7912 | Joint List | IL | Alliance → mapped to Hadash (421), Balad (1663) | Arab party coalition (2015-2021). GPS explicitly names 4 constituents. Hadash and Balad have separate CHES 2022 data and their own text_data. |
+| 3979 | Salvemos a México | MX | Alliance → mapped to PRI (1474), PVEM (446) | PRI-PVEM electoral coalition. Both constituents have extensive text and expert data. |
+
+## Audit Methodology
+
+The audit script `scripts/audit_party_types.py` systematically checks every party in the output CSV:
+
+1. **Union mapping check**: Verifies no `manifesto_pf_id` from `union_mapping.csv` appears in output (hard fail).
+2. **Constituent check**: Identifies parties that are `expert_pf_id` in the mapping (expected: these are individual constituents of unions).
+3. **Expert data check**: Flags parties with no expert survey data (text-only entities).
+4. **Name pattern check**: Scans PartyFacts names for alliance indicators (keywords: alliance, coalition, bloc, front, union, alianza, frente; characters: +, /, &).
+5. **Classification**: Each party gets one of: `individual_party`, `flagged_for_review`, `error_union_in_output`.
+
+**To re-run after data updates:**
+```bash
+python3 scripts/audit_party_types.py
+```
+
+Output: `scripts/party_type_audit.csv` with columns: `party_id`, `name`, `country`, `in_union_mapping_as_union`, `in_union_mapping_as_constituent`, `has_expert_data`, `name_flags`, `classification`, `evidence`.
+
+**Post-estimation verification** (`02_post_estimation.jl`): After extracting estimates, loads all `manifesto_pf_id` values from `union_mapping.csv` and checks none appear in the output `party_id` column. If any do, the script errors with a hard fail.

metadata/data_dictionary.csv

@@ -0,0 +1,28 @@
+file_name,variable_name,label,description,type,allowed_values,range_min,range_max,missing_value_code,unit,scale_direction,constructed_from,construction_rule,notes
+party_2d_election_year_panel_vN.csv.gz,party_id,PartyFacts party identifier,Identifier for the individual party,integer,,,,,identifier,,PartyFacts crosswalk,Assigned during party harmonization,
+party_2d_election_year_panel_vN.csv.gz,party_name_english,English party name,Party name from the harmonized output,string,,,,,name,,Party metadata,,
+party_2d_election_year_panel_vN.csv.gz,party_name_short,Short party name,Short party label from the harmonized output,string,,,,,name,,Party metadata,,May be missing
+party_2d_election_year_panel_vN.csv.gz,country,Country code,ISO2 or historical country-code identifier,string,,,,,identifier,,Source metadata and PartyFacts,,
+party_2d_election_year_panel_vN.csv.gz,year,Calendar year,Calendar year of the election-year estimate,integer,,1944,2025,,year,,Election and model-year metadata,,
+party_2d_election_year_panel_vN.csv.gz,segment_num,Party segment number,Segment number within party after splitting at long evidence gaps,integer,,1,,,segment,,Party history segmentation,Main segment is coded 1,
+party_2d_election_year_panel_vN.csv.gz,union_party_id,Union or alliance PartyFacts identifier,Identifier of parent union or alliance where applicable,integer,,,,,identifier,,Alliance mapping,,Missing if party is not represented through a union or alliance
+party_2d_election_year_panel_vN.csv.gz,in_union,Union membership indicator,Indicator that the row is associated with a union or alliance,boolean,0;1,0,1,,indicator,,Alliance mapping,,
+party_2d_election_year_panel_vN.csv.gz,pervote,Vote share,Vote share at the election year,numeric,,0,100,,percent,,Election metadata,,
+party_2d_election_year_panel_vN.csv.gz,economic_lr,Economic left-right posterior mean,Posterior mean of economic left-right position,numeric,,0,1,,unit interval,0=left; 1=right,Posterior draws,Mean after inverse-logit transformation,
+party_2d_election_year_panel_vN.csv.gz,galtan,Cultural posterior mean,Posterior mean of cultural position,numeric,,0,1,,unit interval,0=cosmopolitan/socially liberal; 1=traditionalist/nationalist,Posterior draws,Mean after inverse-logit transformation,
+party_2d_election_year_panel_vN.csv.gz,economic_lr_se,Economic posterior standard error,Posterior standard deviation for economic_lr,numeric,,0,,,unit interval,,Posterior draws,Standard deviation over posterior draws,
+party_2d_election_year_panel_vN.csv.gz,galtan_se,Cultural posterior standard error,Posterior standard deviation for galtan,numeric,,0,,,unit interval,,Posterior draws,Standard deviation over posterior draws,
+party_2d_election_year_panel_vN.csv.gz,economic_lr_q025,Economic lower posterior interval,2.5 percent posterior quantile for economic_lr,numeric,,0,1,,unit interval,0=left; 1=right,Posterior draws,Quantile over posterior draws,
+party_2d_election_year_panel_vN.csv.gz,economic_lr_q975,Economic upper posterior interval,97.5 percent posterior quantile for economic_lr,numeric,,0,1,,unit interval,0=left; 1=right,Posterior draws,Quantile over posterior draws,
+party_2d_election_year_panel_vN.csv.gz,galtan_q025,Cultural lower posterior interval,2.5 percent posterior quantile for galtan,numeric,,0,1,,unit interval,0=cosmopolitan/socially liberal; 1=traditionalist/nationalist,Posterior draws,Quantile over posterior draws,
+party_2d_election_year_panel_vN.csv.gz,galtan_q975,Cultural upper posterior interval,97.5 percent posterior quantile for galtan,numeric,,0,1,,unit interval,0=cosmopolitan/socially liberal; 1=traditionalist/nationalist,Posterior draws,Quantile over posterior draws,
+party_2d_election_year_panel_vN.csv.gz,election_id,Election identifier,Election identifier where available,string,,,,,identifier,,Election metadata,,Missing where no election identifier is available
+party_2d_election_year_panel_vN.csv.gz,election_date,Election date,Election date where available,date,,,,,date,,Election metadata,,Missing in current processed election metadata
+party_2d_election_year_panel_vN.csv.gz,has_text,Text support indicator,Indicator for direct or nearby text evidence,boolean,0;1,0,1,,indicator,,Source-support construction,Nearby threshold documented in source_support_dictionary.csv,
+party_2d_election_year_panel_vN.csv.gz,has_expert,Expert support indicator,Indicator for direct or nearby expert evidence,boolean,0;1,0,1,,indicator,,Source-support construction,Nearby threshold documented in source_support_dictionary.csv,
+party_2d_election_year_panel_vN.csv.gz,n_text_sources,Number of text sources,Number of distinct text source families contributing direct or nearby evidence,integer,,0,,,count,,Source-support construction,,
+party_2d_election_year_panel_vN.csv.gz,n_expert_sources,Number of expert sources,Number of distinct expert source families contributing direct or nearby evidence,integer,,0,,,count,,Source-support construction,,
+party_2d_election_year_panel_vN.csv.gz,nearest_text_distance,Nearest text distance,Absolute distance in years to nearest text observation used to inform trajectory,numeric,,0,,,years,,Source-support construction,,Missing if no text observation exists for the party-country key
+party_2d_election_year_panel_vN.csv.gz,nearest_expert_distance,Nearest expert distance,Absolute distance in years to nearest expert observation used to inform trajectory,numeric,,0,,,years,,Source-support construction,,Missing if no expert observation exists for the party-country key
+party_2d_election_year_panel_vN.csv.gz,source_support_class,Source-support class,Summary class of text/expert support,string,both_direct_or_nearby; text_only_direct_or_nearby; expert_only_direct_or_nearby; temporal_propagation,,,,categorical,,Source-support construction,,
+party_2d_annual_model_output_vN.csv.gz,all fields,Annual model output fields,Same model-output fields as the production annual party-position file plus no source-support augmentation,mixed,,,,,,,Posterior processing,,Secondary model output

metadata/source_support_dictionary.csv

@@ -0,0 +1,9 @@
+field,value,definition,notes
+direct_support,observation in same calendar year,Observation in the same calendar year as the election-year estimate,
+nearby_support,observation within threshold,Observation within the stated distance threshold of the election-year estimate,
+nearby_support_threshold_years,2,Numerical threshold in years used to classify nearby support,
+temporal_propagation,no nearby observation,No text or expert observation within the nearby threshold,
+source_support_class,both_direct_or_nearby,Both text and expert evidence are direct or nearby,
+source_support_class,text_only_direct_or_nearby,Text evidence is direct or nearby and expert evidence is not,
+source_support_class,expert_only_direct_or_nearby,Expert evidence is direct or nearby and text evidence is not,
+source_support_class,temporal_propagation,Neither text nor expert evidence is direct or nearby,

models/stan_model_2dim_v6.stan

@@ -0,0 +1,595 @@
+// =============================================================================
+// 2D BIPOLAR LATENT TRAIT MODEL V6 - Hierarchical L-R Weights
+// =============================================================================
+//
+// V6 CHANGE: Hierarchical L-R weights varying by country, source, and decade
+//   - Replaces global simplex[2] lr_weights with additive logit-scale model
+//   - logit_weight[n] = global + country_offset[c] + source_offset[k] + decade_offset[d]
+//   - All offsets use non-centered parameterization with estimated sigma hyperparameters
+//   - Data-poor contexts shrink toward global mean (~55 new scalar parameters)
+//
+// PRESERVED FROM V5:
+//   - Beta-Binomial expert likelihood with K-scaling
+//   - V-Party cultural expansion (5 native items + v2pawelf)
+//   - Mean-constituent model (individual party estimates for union members)
+//   - Segment-based indexing (S segments, R segment-years)
+//   - 3-level hierarchical variance (global -> country -> family)
+//   - Random walk dynamics within segments
+//   - CDU anchor for scale identification (CDU=1375, not CDU/CSU=211)
+//   - Non-centered parameterization for efficiency
+//   - Zero-inflation model for manifesto data
+//   - Country-item intercepts (coding convention differences across countries)
+//   - Binomial-logit likelihood for text data (unchanged)
+//
+// =============================================================================
+
+data {
+  // Segment structure
+  int<lower=1> S;                      // Number of segments
+  int<lower=1> P;                      // Number of countries
+  int<lower=1> R;                      // Total unique segment-year combinations
+  int<lower=1> T_year;                 // Total number of years
+  array[S] int<lower=1> len_theta_ts;  // Number of years per segment
+
+  // Country membership for each segment
+  array[S] int<lower=1, upper=P> segment_country;
+
+  // Segment family data
+  int<lower=1> F;                                 // Number of party families
+  array[S] int<lower=1, upper=F> segment_family;  // Family index for each segment
+
+  // =========================================================================
+  // Text data (manifesto + PolDem)
+  // =========================================================================
+  int<lower=1> N_man;                  // Number of text observations
+  int<lower=1> K_man;                  // Number of unique text items
+  array[N_man] int<lower=1, upper=K_man> kk_man;   // Text item index
+  array[N_man] int<lower=1, upper=S> ss_man;       // Segment index (first constituent for unions)
+  array[N_man] int<lower=1, upper=P> pp_man;       // Country index
+  array[N_man] int<lower=0> positive;              // Positive mentions
+  array[N_man] int<lower=0> sample;                // Total sample size
+  array[N_man] int<lower=1, upper=T_year> year_for_man; // Year index
+
+  // Dimension and direction for text data
+  array[N_man] int<lower=1, upper=2> dim_idx_man;  // 1=economic, 2=galtan
+  array[N_man] int<lower=-1, upper=1> direction_man; // +1=right/TAN, -1=left/GAL
+
+  // V4: Constituent structure for manifesto observations
+  int<lower=1> N_const_man_total;                                    // Total entries in const_rr_man
+  array[N_man] int<lower=1> n_const_man;                             // Number of constituents per obs
+  array[N_man] int<lower=1> const_offset_man;                        // Offset into const_rr_man
+  array[N_const_man_total] int<lower=1, upper=R> const_rr_man;       // Constituent rr indices
+
+  // Country-item-year data (used by zero-inflation model)
+  int<lower=1> N_ciy;                  // Number of unique country-item-year combinations
+  array[N_man] int<lower=1, upper=N_ciy> ciy_idx;
+
+  // =========================================================================
+  // Expert dimension-specific data (V5: integer observations + scale size)
+  // =========================================================================
+  int<lower=1> N_exp_dim;                  // Number of dimension-specific expert observations
+  int<lower=1> K_exp_dim;                  // Number of unique expert items
+  array[N_exp_dim] int<lower=1, upper=K_exp_dim> kk_exp_dim;
+  array[N_exp_dim] int<lower=1, upper=S> ss_exp_dim;
+  array[N_exp_dim] int<lower=1, upper=P> pp_exp_dim;
+  array[N_exp_dim] int<lower=0> val_dim_int;           // V5: rounded sum = round(mean * K * n_scale)
+  array[N_exp_dim] int<lower=1> n_total_exp_dim;       // V5: K * n_scale (total trials)
+  array[N_exp_dim] int<lower=1> n_experts_exp_dim;     // V5: K (number of experts)
+
+  // Dimension index for expert data
+  array[N_exp_dim] int<lower=1, upper=2> dim_idx_exp;  // 1=economic, 2=galtan
+
+  // V4: Constituent structure for expert dim observations
+  int<lower=1> N_const_exp_dim_total;
+  array[N_exp_dim] int<lower=1> n_const_exp_dim;
+  array[N_exp_dim] int<lower=1> const_offset_exp_dim;
+  array[N_const_exp_dim_total] int<lower=1, upper=R> const_rr_exp_dim;
+
+  // =========================================================================
+  // Expert general L-R data (V5: integer observations + scale size)
+  // =========================================================================
+  int<lower=1> N_exp_lr;
+  int<lower=1> K_exp_lr;
+  array[N_exp_lr] int<lower=1, upper=K_exp_lr> kk_exp_lr;
+  array[N_exp_lr] int<lower=1, upper=S> ss_exp_lr;
+  array[N_exp_lr] int<lower=1, upper=P> pp_exp_lr;
+  array[N_exp_lr] int<lower=0> val_lr_int;             // V5: rounded sum = round(mean * K * n_scale)
+  array[N_exp_lr] int<lower=1> n_total_exp_lr;         // V5: K * n_scale (total trials)
+  array[N_exp_lr] int<lower=1> n_experts_exp_lr;       // V5: K (number of experts)
+
+  // V6: Decade indexing for hierarchical L-R weights
+  int<lower=1> D_lr;                                   // Number of decades
+  array[N_exp_lr] int<lower=1, upper=D_lr> dd_exp_lr;  // Decade index per LR obs
+
+  // V4: Constituent structure for expert L-R observations
+  int<lower=1> N_const_exp_lr_total;
+  array[N_exp_lr] int<lower=1> n_const_exp_lr;
+  array[N_exp_lr] int<lower=1> const_offset_exp_lr;
+  array[N_const_exp_lr_total] int<lower=1, upper=R> const_rr_exp_lr;
+
+  // Prior information
+  real mn_resp_log_man;
+  real mn_resp_log_exp_dim;
+  real mn_resp_log_exp_lr;
+
+  // Identification anchor
+  int<lower=1, upper=S> anchor_segment;  // CDU segment (1375 with unions, 211 without)
+
+  // V3 compatibility: these are still passed but not used in V4+ likelihood
+  // (kept so older data dicts work without modification for backwards compat)
+  array[N_man] int<lower=1, upper=R> rr_man;       // Segment-year index (unused in V4+ likelihood)
+  array[N_exp_dim] int<lower=1, upper=R> rr_exp_dim;
+  array[N_exp_lr] int<lower=1, upper=R> rr_exp_lr;
+}
+
+transformed data {
+  real eps = 1e-6;
+  real one_minus_eps = 1 - eps;
+
+  // Count zero and non-zero samples
+  int N_man_zero = 0;
+  int N_man_nonzero = 0;
+  for (n in 1:N_man) {
+    if (sample[n] == 0) {
+      N_man_zero += 1;
+    } else {
+      N_man_nonzero += 1;
+    }
+  }
+
+  // Create index arrays for zero/nonzero split
+  array[N_man_zero > 0 ? N_man_zero : 1] int idx_zero;
+  array[N_man_nonzero > 0 ? N_man_nonzero : 1] int idx_nonzero;
+
+  {
+    int pos_zero = 1;
+    int pos_nonzero = 1;
+    for (n in 1:N_man) {
+      if (sample[n] == 0) {
+        idx_zero[pos_zero] = n;
+        pos_zero += 1;
+      } else {
+        idx_nonzero[pos_nonzero] = n;
+        pos_nonzero += 1;
+      }
+    }
+  }
+
+  // V4: Pre-compute constituent info for nonzero manifesto obs
+  array[N_man_nonzero > 0 ? N_man_nonzero : 1] int kk_man_nonzero;
+  array[N_man_nonzero > 0 ? N_man_nonzero : 1] int orig_idx_nonzero;
+  array[N_man_nonzero > 0 ? N_man_nonzero : 1] int direction_nonzero;
+  array[N_man_nonzero > 0 ? N_man_nonzero : 1] int pp_man_nonzero;
+  array[N_man_nonzero > 0 ? N_man_nonzero : 1] int dim_idx_nonzero;
+  array[N_man_nonzero > 0 ? N_man_nonzero : 1] int n_const_nonzero;
+  array[N_man_nonzero > 0 ? N_man_nonzero : 1] int const_offset_nonzero;
+
+  {
+    for (i in 1:N_man_nonzero) {
+      int n = idx_nonzero[i];
+      orig_idx_nonzero[i] = n;
+      kk_man_nonzero[i] = kk_man[n];
+      direction_nonzero[i] = direction_man[n];
+      pp_man_nonzero[i] = pp_man[n];
+      dim_idx_nonzero[i] = dim_idx_man[n];
+      n_const_nonzero[i] = n_const_man[n];
+      const_offset_nonzero[i] = const_offset_man[n];
+    }
+  }
+
+  // Pre-compute segment start positions
+  array[S + 1] int segment_start;
+  segment_start[1] = 1;
+  for (s in 1:S) {
+    segment_start[s + 1] = segment_start[s] + len_theta_ts[s];
+  }
+
+  // Extract sample and positive for nonzero observations
+  array[N_man_nonzero > 0 ? N_man_nonzero : 1] int sample_nonzero;
+  array[N_man_nonzero > 0 ? N_man_nonzero : 1] int positive_nonzero;
+  for (i in 1:N_man_nonzero) {
+    sample_nonzero[i] = sample[idx_nonzero[i]];
+    positive_nonzero[i] = positive[idx_nonzero[i]];
+  }
+
+  // Pre-compute family-to-country mapping
+  array[F] int family_country;
+  {
+    for (f in 1:F) {
+      family_country[f] = 0;
+    }
+    for (s in 1:S) {
+      int f = segment_family[s];
+      if (family_country[f] == 0) {
+        family_country[f] = segment_country[s];
+      }
+    }
+    for (f in 1:F) {
+      if (family_country[f] == 0) {
+        family_country[f] = 1;
+      }
+    }
+  }
+}
+
+parameters {
+  // =========================================================================
+  // Latent position parameters - 2 dimensions
+  // =========================================================================
+  matrix[2, R] theta_ncp;                 // Non-centered: [1]=economic_lr, [2]=galtan
+  matrix[2, S] theta_init_raw;            // Initial traits per segment
+  vector<lower=0>[2] sigma_theta_init;    // SD of initial theta per dimension
+
+  // =========================================================================
+  // Three-level hierarchical variance
+  // =========================================================================
+  vector[2] mu_sigma_global_raw;          // Global mean RW variance
+  vector<lower=0>[2] tau_sigma_country;   // Country deviation scale
+  matrix[2, P] sigma_country_raw;         // Non-centered country deviations
+  vector<lower=0>[2] tau_sigma_family;    // Family deviation scale
+  matrix[2, F] sigma_family_raw;          // Non-centered family deviations
+
+  // =========================================================================
+  // Country-item intercepts (coding convention differences)
+  // =========================================================================
+  matrix[P, K_man] country_item_raw;
+  real<lower=0> sigma_country_item;
+
+  // =========================================================================
+  // Zero-sample parameters
+  // =========================================================================
+  real alpha_zs;
+  vector[T_year] year_effect_raw;
+  real<lower=0> sigma_year_effect;
+  vector[S] segment_zs_raw;
+  real<lower=0> sigma_segment_zs;
+  vector[N_ciy] ciy_zs_raw;
+  real<lower=0> sigma_ciy_zs;
+
+  // =========================================================================
+  // Item parameters
+  // =========================================================================
+
+  // Text data: intercept + single positive loading (direction handled in data)
+  vector[K_man] gamma_man_intercept_raw;
+  vector<lower=0>[K_man] gamma_man_loading;  // Positive loading
+
+  // Expert dimension-specific: intercept + slope (direct mapping to dimension)
+  vector[K_exp_dim] gamma_exp_intercept_raw;
+  vector<lower=0>[K_exp_dim] gamma_exp_slope;
+
+  // Expert general L-R: intercept + slope
+  vector[K_exp_lr] gamma_lr_intercept_raw;
+  vector<lower=0>[K_exp_lr] gamma_lr_slope;
+
+  // V6: Hierarchical L-R weights (replace simplex[2] lr_weights)
+  real lr_weight_global;                       // Global logit-scale weight
+  vector[P] lr_country_offset_raw;             // Country offsets (non-centered)
+  vector[K_exp_lr] lr_source_offset_raw;       // Source offsets (non-centered)
+  vector[D_lr] lr_decade_offset_raw;           // Decade offsets (non-centered)
+  real<lower=0> sigma_lr_country;              // SD of country offsets
+  real<lower=0> sigma_lr_source;               // SD of source offsets
+  real<lower=0> sigma_lr_decade;               // SD of decade offsets
+
+  // Precision parameters
+  real<lower=0> phi_exp_dim;  // Precision for dimension-specific (now: only measurement noise)
+  real<lower=0> phi_exp_lr;   // Precision for general L-R (now: only measurement noise)
+
+  // Scale parameters
+  real<lower=0> sigma_intercept_man;
+  real<lower=0> sigma_loading_man;
+  real<lower=0> sigma_intercept_exp;
+  real<lower=0> sigma_slope_exp;
+  real<lower=0> sigma_intercept_lr;
+  real<lower=0> sigma_slope_lr;
+  real mu_lambda_man;
+  real mu_lambda_exp_dim;
+  real mu_lambda_exp_lr;
+}
+
+transformed parameters {
+  matrix[2, R] theta;                  // [1]=economic_lr, [2]=galtan
+  matrix[2, S] theta_init;
+
+  // Three-level hierarchical variance (2D)
+  matrix[2, P] sigma_theta_country;
+  for (d in 1:2) {
+    for (p in 1:P) {
+      sigma_theta_country[d, p] = log1p_exp(
+        mu_sigma_global_raw[d] + tau_sigma_country[d] * sigma_country_raw[d, p]
+      );
+    }
+  }
+
+  matrix[2, F] sigma_theta_family;
+  for (d in 1:2) {
+    for (f in 1:F) {
+      int c = family_country[f];
+      sigma_theta_family[d, f] = log1p_exp(
+        log(sigma_theta_country[d, c]) + tau_sigma_family[d] * sigma_family_raw[d, f]
+      );
+    }
+  }
+
+  // Non-centered parameterization for theta_init
+  for (dim in 1:2) {
+    theta_init[dim, :] = sigma_theta_init[dim] * theta_init_raw[dim, :];
+  }
+
+  // Country-item intercepts
+  matrix[P, K_man] country_item_intercept = sigma_country_item * country_item_raw;
+
+  // Zero-sample components
+  vector[T_year] year_effect = sigma_year_effect * year_effect_raw;
+  vector[S] segment_zs = sigma_segment_zs * segment_zs_raw;
+  vector[N_ciy] ciy_zs = sigma_ciy_zs * ciy_zs_raw;
+
+  // Zero-sample probability
+  vector[N_man] zero_sample_logit = alpha_zs +
+                                     year_effect[year_for_man] +
+                                     segment_zs[ss_man] +
+                                     ciy_zs[ciy_idx];
+  vector[N_man] zero_sample_prob = inv_logit(zero_sample_logit);
+  zero_sample_prob = fmax(fmin(zero_sample_prob, one_minus_eps), eps);
+
+  // Construct theta using family-specific random walk variance (2D)
+  for (dim in 1:2) {
+    for (s in 1:S) {
+      int start = segment_start[s];
+      int Ts = len_theta_ts[s];
+      int fam = segment_family[s];
+      real sigma_s = sigma_theta_family[dim, fam];
+
+      theta[dim, start] = theta_init[dim, s] + sigma_s * theta_ncp[dim, start];
+      if (Ts > 1) {
+        theta[dim, start + 1 : start + Ts - 1] = theta[dim, start] +
+          cumulative_sum(sigma_s * theta_ncp[dim, start + 1 : start + Ts - 1]);
+      }
+    }
+  }
+
+  // Item parameters
+  vector[K_man] gamma_man_intercept = mu_lambda_man + sigma_intercept_man * gamma_man_intercept_raw;
+  vector[K_exp_dim] gamma_exp_intercept = mu_lambda_exp_dim + sigma_intercept_exp * gamma_exp_intercept_raw;
+  vector[K_exp_lr] gamma_lr_intercept = mu_lambda_exp_lr + sigma_intercept_lr * gamma_lr_intercept_raw;
+}
+
+model {
+  // =========================================================================
+  // PRIORS
+  // =========================================================================
+
+  // Three-level hierarchical variance priors (2D)
+  mu_sigma_global_raw ~ normal(-0.8, 0.5);
+  tau_sigma_country ~ normal(0, 0.3);
+  to_vector(sigma_country_raw) ~ std_normal();
+  tau_sigma_family ~ normal(0, 0.2);
+  to_vector(sigma_family_raw) ~ std_normal();
+
+  // Other variance priors
+  sigma_theta_init ~ normal(0, 0.5);
+  to_vector(theta_init_raw) ~ std_normal();
+  to_vector(theta_ncp) ~ std_normal();
+
+  // Country-item intercept priors
+  to_vector(country_item_raw) ~ std_normal();
+  sigma_country_item ~ normal(0, 0.3);
+
+  // Zero-sample priors
+  alpha_zs ~ normal(-1, 1);
+  year_effect_raw ~ std_normal();
+  sigma_year_effect ~ normal(0, 0.5);
+  segment_zs_raw ~ std_normal();
+  sigma_segment_zs ~ normal(0, 0.3);
+  ciy_zs_raw ~ std_normal();
+  sigma_ciy_zs ~ normal(0, 0.3);
+
+  // =========================================================================
+  // Item parameter priors
+  // =========================================================================
+
+  // Text data item priors
+  mu_lambda_man ~ normal(mn_resp_log_man, 0.5);
+  sigma_intercept_man ~ normal(0, 1);
+  sigma_loading_man ~ normal(0, 0.5);
+  gamma_man_intercept_raw ~ std_normal();
+  gamma_man_loading ~ normal(1.0, sigma_loading_man);
+
+  // Expert dimension item priors
+  mu_lambda_exp_dim ~ normal(mn_resp_log_exp_dim, 0.5);
+  sigma_intercept_exp ~ normal(0, 1);
+  sigma_slope_exp ~ normal(0, 0.5);
+  gamma_exp_intercept_raw ~ std_normal();
+  gamma_exp_slope ~ normal(1.0, sigma_slope_exp);
+
+  // Expert L-R item priors
+  mu_lambda_exp_lr ~ normal(mn_resp_log_exp_lr, 0.5);
+  sigma_intercept_lr ~ normal(0, 1);
+  sigma_slope_lr ~ normal(0, 0.5);
+  gamma_lr_intercept_raw ~ std_normal();
+  gamma_lr_slope ~ normal(1.0, sigma_slope_lr);
+
+  // V6: Hierarchical L-R weight priors
+  lr_weight_global ~ normal(0, 1);
+  lr_country_offset_raw ~ std_normal();
+  lr_source_offset_raw ~ std_normal();
+  lr_decade_offset_raw ~ std_normal();
+  sigma_lr_country ~ normal(0, 0.5);
+  sigma_lr_source ~ normal(0, 0.5);
+  sigma_lr_decade ~ normal(0, 0.5);
+
+  // Expert data precision priors
+  phi_exp_dim ~ gamma(50, 0.5);
+  phi_exp_lr ~ gamma(10, 0.5);
+
+  // =========================================================================
+  // CDU ANCHOR CONSTRAINT
+  // With unions: anchors CDU (1375) at moderate center-right position
+  // Without unions: anchors CDU/CSU (211) as before
+  // =========================================================================
+  target += normal_lpdf(theta_init[1, anchor_segment] | 0.2, 0.2);  // economic_lr
+  target += normal_lpdf(theta_init[2, anchor_segment] | 0.2, 0.2);  // galtan
+
+  // =========================================================================
+  // LIKELIHOOD 1: Text data (binomial with zero-inflation)
+  // V4: Mean-constituent averaging for union observations
+  // =========================================================================
+
+  // Zero-sample observations
+  if (N_man_zero > 0) {
+    target += sum(log(zero_sample_prob[idx_zero]));
+  }
+
+  // Non-zero observations with constituent averaging
+  if (N_man_nonzero > 0) {
+    vector[N_man_nonzero] lin_man;
+
+    for (i in 1:N_man_nonzero) {
+      int nc = n_const_nonzero[i];
+      int off = const_offset_nonzero[i];
+      int dim = dim_idx_nonzero[i];
+      real avg_pos;
+
+      if (nc == 1) {
+        // Fast path: single party (>95% of observations)
+        avg_pos = theta[dim, const_rr_man[off]];
+      } else {
+        // Union: average over constituent thetas
+        avg_pos = 0;
+        for (c in 0:(nc-1)) {
+          avg_pos += theta[dim, const_rr_man[off + c]];
+        }
+        avg_pos /= nc;
+      }
+
+      lin_man[i] = gamma_man_intercept[kk_man_nonzero[i]] +
+                   direction_nonzero[i] * gamma_man_loading[kk_man_nonzero[i]] * avg_pos +
+                   country_item_intercept[pp_man_nonzero[i], kk_man_nonzero[i]];
+    }
+
+    for (i in 1:N_man_nonzero) {
+      target += log1m(zero_sample_prob[orig_idx_nonzero[i]]) +
+                binomial_logit_lpmf(positive_nonzero[i] | sample_nonzero[i], lin_man[i]);
+    }
+  }
+
+  // =========================================================================
+  // LIKELIHOOD 2: Expert dimension-specific data (V5: beta-binomial likelihood)
+  // V4: Constituent averaging for union-level expert obs
+  // =========================================================================
+  {
+    vector[N_exp_dim] pos;
+    for (n in 1:N_exp_dim) {
+      int nc = n_const_exp_dim[n];
+      int off = const_offset_exp_dim[n];
+      int dim = dim_idx_exp[n];
+
+      if (nc == 1) {
+        pos[n] = theta[dim, const_rr_exp_dim[off]];
+      } else {
+        pos[n] = 0;
+        for (c in 0:(nc-1)) {
+          pos[n] += theta[dim, const_rr_exp_dim[off + c]];
+        }
+        pos[n] /= nc;
+      }
+    }
+
+    vector[N_exp_dim] lin_exp_dim;
+    for (n in 1:N_exp_dim) {
+      lin_exp_dim[n] = gamma_exp_intercept[kk_exp_dim[n]] +
+                       gamma_exp_slope[kk_exp_dim[n]] * pos[n];
+    }
+
+    vector[N_exp_dim] mu_exp_dim = inv_logit(lin_exp_dim);
+    mu_exp_dim = fmax(fmin(mu_exp_dim, one_minus_eps), eps);
+
+    // K-scaling: phi * K corrects for independent expert perceptions
+    vector[N_exp_dim] alpha_exp_dim = phi_exp_dim * to_vector(n_experts_exp_dim) .* mu_exp_dim;
+    vector[N_exp_dim] beta_exp_dim = phi_exp_dim * to_vector(n_experts_exp_dim) .* (1 - mu_exp_dim);
+    val_dim_int ~ beta_binomial(n_total_exp_dim, alpha_exp_dim, beta_exp_dim);
+  }
+
+  // =========================================================================
+  // LIKELIHOOD 3: Expert general L-R data (V6: hierarchical per-obs weights)
+  // V4: Constituent averaging + weighted combination of both dimensions
+  // V6: Per-observation weights via country + source + decade offsets
+  // =========================================================================
+  {
+    // V6: Compute per-observation economic weight on logit scale
+    vector[N_exp_lr] logit_w;
+    for (n in 1:N_exp_lr) {
+      logit_w[n] = lr_weight_global
+                 + sigma_lr_country * lr_country_offset_raw[pp_exp_lr[n]]
+                 + sigma_lr_source  * lr_source_offset_raw[kk_exp_lr[n]]
+                 + sigma_lr_decade  * lr_decade_offset_raw[dd_exp_lr[n]];
+    }
+    vector[N_exp_lr] w_econ = inv_logit(logit_w);
+
+    vector[N_exp_lr] combined_pos;
+    for (n in 1:N_exp_lr) {
+      int nc = n_const_exp_lr[n];
+      int off = const_offset_exp_lr[n];
+
+      if (nc == 1) {
+        int r = const_rr_exp_lr[off];
+        combined_pos[n] = w_econ[n] * theta[1, r] + (1 - w_econ[n]) * theta[2, r];
+      } else {
+        // Average the combined position across constituents
+        combined_pos[n] = 0;
+        for (c in 0:(nc-1)) {
+          int r = const_rr_exp_lr[off + c];
+          combined_pos[n] += w_econ[n] * theta[1, r] + (1 - w_econ[n]) * theta[2, r];
+        }
+        combined_pos[n] /= nc;
+      }
+    }
+
+    vector[N_exp_lr] lin_exp_lr;
+    for (n in 1:N_exp_lr) {
+      lin_exp_lr[n] = gamma_lr_intercept[kk_exp_lr[n]] +
+                      gamma_lr_slope[kk_exp_lr[n]] * combined_pos[n];
+    }
+
+    vector[N_exp_lr] mu_exp_lr = inv_logit(lin_exp_lr);
+    mu_exp_lr = fmax(fmin(mu_exp_lr, one_minus_eps), eps);
+
+    // K-scaling: phi * K corrects for independent expert perceptions
+    vector[N_exp_lr] alpha_exp_lr = phi_exp_lr * to_vector(n_experts_exp_lr) .* mu_exp_lr;
+    vector[N_exp_lr] beta_exp_lr = phi_exp_lr * to_vector(n_experts_exp_lr) .* (1 - mu_exp_lr);
+    val_lr_int ~ beta_binomial(n_total_exp_lr, alpha_exp_lr, beta_exp_lr);
+  }
+}
+
+generated quantities {
+  // =========================================================================
+  // Direct outputs (no derivation needed)
+  // =========================================================================
+
+  // Two bipolar scales on [0,1] via inv_logit
+  vector[R] economic_lr = inv_logit(to_vector(theta[1, :]));  // 0=left, 1=right
+  vector[R] galtan = inv_logit(to_vector(theta[2, :]));       // 0=GAL, 1=TAN
+
+  // General left-right combining both dimensions (using global weight only)
+  // Per-R country/source/decade info not available in GQ, so use global mean
+  real lr_w_econ_global = inv_logit(lr_weight_global);
+  vector[R] general_lr = inv_logit(
+    lr_w_econ_global * to_vector(theta[1, :]) + (1 - lr_w_econ_global) * to_vector(theta[2, :])
+  );
+
+  // V6: Expose hierarchical L-R weight diagnostics
+  real lr_weight_econ_global = lr_w_econ_global;
+  real lr_sigma_country = sigma_lr_country;
+  real lr_sigma_source = sigma_lr_source;
+  real lr_sigma_decade = sigma_lr_decade;
+
+  // Expose hierarchical variance diagnostics (2D)
+  vector[2] mean_sigma_global;
+  vector[2] mean_sigma_country;
+  vector[2] mean_sigma_family;
+  for (d in 1:2) {
+    mean_sigma_global[d] = log1p_exp(mu_sigma_global_raw[d]);
+    mean_sigma_country[d] = mean(sigma_theta_country[d, :]);
+    mean_sigma_family[d] = mean(sigma_theta_family[d, :]);
+  }
+}

run_estimation.sh

@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+MODE="${1:-reuse}"
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd -P)"
+cd "$repo_root"
+
+case "$MODE" in
+  full|reuse|dry-run) ;;
+  *)
+    echo "Usage: $0 [full|reuse|dry-run]" >&2
+    exit 1
+    ;;
+esac
+
+export PARTY2D_RAW_DATA_DIR="${PARTY2D_RAW_DATA_DIR:-$repo_root/_local/raw}"
+export TMPDIR="${TMPDIR:-$repo_root/_local/tmp}"
+mkdir -p "$TMPDIR"
+
+if [ "$MODE" = "dry-run" ]; then
+  echo "Checking commands..."
+  command -v bash >/dev/null
+  command -v Rscript >/dev/null
+  command -v julia >/dev/null
+  echo "Checking key files..."
+  test -f Project.toml
+  test -f Manifest.toml
+  test -f models/stan_model_2dim_v6.stan
+  test -f data/text_data.csv
+  test -f data/expert.csv
+  test -f data/lr_data.csv
+  test -f data/model_data.csv
+  echo "Checking shell syntax..."
+  bash -n scripts/01_prepare_data.sh
+  bash -n scripts/02_fit_model.sh
+  bash -n scripts/03_extract_estimates.sh
+  bash -n scripts/04_enrich_estimates.sh
+  bash -n scripts/05_validate_estimates.sh
+  echo "Checking R syntax..."
+  Rscript -e 'files <- list.files("src/r", pattern="\\.R$", full.names=TRUE); invisible(lapply(files, parse))'
+  echo "Checking Julia project can instantiate without running model code..."
+  julia --project=. -e 'import Pkg; Pkg.instantiate(); println("Julia project OK")'
+  echo "Dry run passed. No model fitting was run."
+  exit 0
+fi
+
+bash scripts/01_prepare_data.sh
+
+if [ "$MODE" = "full" ]; then
+  bash scripts/02_fit_model.sh
+else
+  echo "reuse: skipping Stan model fitting; using latest existing model output"
+fi
+
+bash scripts/03_extract_estimates.sh
+bash scripts/04_enrich_estimates.sh

scripts/01_prepare_data.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+cd "$repo_root"
+
+Rscript -e 'setwd("data"); source("../src/r/00_data-management.R")'

scripts/02_fit_model.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+cd "$repo_root"
+
+julia --project=. src/julia/01_run_model.jl

scripts/03_extract_estimates.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+cd "$repo_root"
+
+julia --project=. src/julia/02_post_estimation.jl

scripts/04_enrich_estimates.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+cd "$repo_root"
+
+julia --project=. src/julia/02_enrich_output.jl

scripts/05_validate_estimates.sh

@@ -0,0 +1,9 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd -P)"
+cd "$repo_root"
+
+julia --project=. src/julia/validate_convergent.jl
+julia --project=. src/julia/validate_uncertainty.jl
+julia --project=. src/julia/validate_construct.jl

src/julia/01_run_model.jl

@@ -0,0 +1,340 @@
+#!/usr/bin/env julia
+#############################################################################
+## run_model.jl
+## Main runner for latent trait model
+## Executes the complete pipeline: data loading → preparation → model fitting
+##
+## Supports two model versions:
+##   - "2dim": 2D bipolar model (V1) - estimates economic_lr and galtan directly
+##   - "4dim": 4D unipolar model (V10) - estimates 4 traits, derives 2 scales
+##
+## Default is 2D model (better identification, faster convergence)
+#############################################################################
+
+using Dates
+
+#############################################################################
+## EXECUTION CONFIGURATION (Change these values as needed)
+#############################################################################
+const MODEL_VERSION = "2dim"                                # "2dim" (recommended) or "4dim"
+const STAN_MODEL_FILE = MODEL_VERSION == "2dim" ? "models/stan_model_2dim_v6.stan" : "models/stan_model_4dim_v10.stan"
+const NUM_CHAINS = 4                                        # Number of MCMC chains (run in parallel)
+const NUM_WARMUP = 1000                                     # Number of warmup iterations
+const NUM_SAMPLES = 2000                                    # Number of sampling iterations
+const ADAPT_DELTA = 0.95                                    # Target acceptance probability
+const MAX_DEPTH = 15                                        # Maximum tree depth
+const START_YEAR = 1944                                     # First year to include (avoid sparse early data)
+
+println("=" ^ 70)
+println("Latent Trait Model - Estimation Pipeline")
+println("=" ^ 70)
+println("Started at: ", Dates.now())
+println("Configuration:")
+println("  Model version: $(MODEL_VERSION)")
+println("  Stan model: $(STAN_MODEL_FILE)")
+println("  Chains: $(NUM_CHAINS)")
+println("  Warmup iterations: $(NUM_WARMUP)")
+println("  Sampling iterations: $(NUM_SAMPLES)")
+println("  Total iterations per chain: $(NUM_WARMUP + NUM_SAMPLES)")
+println("  Adapt delta: $(ADAPT_DELTA)")
+println("  Max depth: $(MAX_DEPTH)")
+println("  Start year: $(START_YEAR)")
+if MODEL_VERSION == "2dim"
+    println("\n  2D MODEL: Estimates economic_lr and galtan directly")
+    println("           (Half the parameters, better convergence)")
+else
+    println("\n  4D MODEL: Estimates 4 traits, derives 2 scales")
+    println("           (Known identification issues - see VERSION_HISTORY.md)")
+end
+println("=" ^ 70)
+
+# Include pipeline modules
+include("pipeline/00_validation.jl")        # Validation checks
+include("pipeline/02_data_loading.jl")
+include("pipeline/03_data_preparation.jl")
+include("pipeline/04_model_execution.jl")
+include("pipeline/05_results_processing.jl")
+
+# Load robust save module
+include("pipeline/06_save_model.jl")
+import .RobustSave: robust_save_model
+
+function run_model(;
+    num_chains=NUM_CHAINS,
+    num_warmup=NUM_WARMUP,
+    num_samples=NUM_SAMPLES,
+    adapt_delta=ADAPT_DELTA,
+    max_depth=MAX_DEPTH,
+    model_file=STAN_MODEL_FILE,
+    start_year=START_YEAR,
+    data_dir="data"
+)
+    """Run the complete latent trait model pipeline (2D or 4D based on MODEL_VERSION)"""
+
+    try
+        # Step 1: Load and preprocess data
+        println("\n" * "="^50)
+        println("STEP 1: DATA LOADING")
+        println("="^50)
+
+        manifesto, expert_dim, expert_lr, year0, union_to_constituents, constituent_to_union = load_and_preprocess_4dim_data(start_year; data_dir=data_dir)
+
+        # Step 2: Prepare Stan data structure
+        println("\n" * "="^50)
+        println("STEP 2: DATA PREPARATION")
+        println("="^50)
+
+        # Prepare indices and mappings (V4: union-aware)
+        data_prep = prepare_4dim_stan_data(manifesto, expert_dim, expert_lr, year0;
+                                           union_to_constituents=union_to_constituents,
+                                           constituent_to_union=constituent_to_union)
+
+        # Finalize Stan data dictionary (V4: includes constituent arrays)
+        final_data = finalize_4dim_stan_data(
+            data_prep.manifesto, data_prep.expert_dim, data_prep.expert_lr,
+            data_prep.segment_year, data_prep.segment_info,
+            data_prep.all_parties, data_prep.all_groups,
+            data_prep.group_to_index, year0, data_prep.S, data_prep.J, data_prep.P, data_prep.R,
+            data_prep.N_ciy, data_prep.len_theta_ts, data_prep.segment_country_idx,
+            data_prep.F, data_prep.segment_family_idx, data_prep.anchor_segment_idx;
+            N_const_man_total=data_prep.N_const_man_total,
+            n_const_man=data_prep.n_const_man,
+            const_offset_man=data_prep.const_offset_man,
+            const_rr_man=data_prep.const_rr_man,
+            N_const_exp_dim_total=data_prep.N_const_exp_dim_total,
+            n_const_exp_dim=data_prep.n_const_exp_dim,
+            const_offset_exp_dim=data_prep.const_offset_exp_dim,
+            const_rr_exp_dim=data_prep.const_rr_exp_dim,
+            N_const_exp_lr_total=data_prep.N_const_exp_lr_total,
+            n_const_exp_lr=data_prep.n_const_exp_lr,
+            const_offset_exp_lr=data_prep.const_offset_exp_lr,
+            const_rr_exp_lr=data_prep.const_rr_exp_lr
+        )
+
+        dat_4dim = final_data.dat_4dim
+
+        # Step 3: Validate data BEFORE running Stan
+        println("\n" * "="^50)
+        println("STEP 3: DATA VALIDATION")
+        println("="^50)
+
+        if !validate_stan_data(dat_4dim; verbose=true)
+            error("Data validation failed - see errors above")
+        end
+
+        estimate_memory_requirements(dat_4dim; verbose=true)
+
+        # Step 4: Create initialization function
+        println("\n" * "="^50)
+        println("STEP 4: MODEL INITIALIZATION")
+        println("="^50)
+
+        # Determine model version for initialization
+        model_init_version = MODEL_VERSION == "2dim" ? "v1_2dim" : "v10"
+
+        # Use S (segments) for initialization, not J (parties)
+        init_fn = create_init_function(dat_4dim, data_prep.S, data_prep.P,
+                                       data_prep.R, final_data.T_year, data_prep.N_ciy;
+                                       model_version=model_init_version)
+
+        # Validate initialization values
+        println("\nValidating initialization for chain 1...")
+        test_init = init_fn()
+        if !validate_init_values(test_init; verbose=true)
+            error("Initialization validation failed - see errors above")
+        end
+
+        # Step 5: Run Stan model
+        println("\n" * "="^50)
+        println("STEP 5: MODEL EXECUTION")
+        println("="^50)
+
+        # Create temp folder for output
+        temp_folder = mktempdir()
+        println("Temporary folder for Stan output: $temp_folder")
+
+        # Run Stan model
+        stanmodel = run_4dim_stan_model(
+            dat_4dim, init_fn, temp_folder;
+            num_chains=num_chains,
+            num_warmup=num_warmup,
+            num_samples=num_samples,
+            adapt_delta=adapt_delta,
+            max_depth=max_depth,
+            model_file=model_file
+        )
+
+        # Step 6: Results Processing & Diagnostics
+        println("\n" * "="^50)
+        println("STEP 6: RESULTS PROCESSING & DIAGNOSTICS")
+        println("="^50)
+
+        results = extract_model_results_4dim(stanmodel)
+        diagnostics = compute_model_diagnostics(stanmodel)
+
+        println("\nCONVERGENCE DIAGNOSTICS:")
+        println("  Max R-hat: $(round(diagnostics.max_rhat, digits=4))")
+        println("  Mean R-hat: $(round(diagnostics.mean_rhat, digits=4))")
+        println("  High R-hat count: $(diagnostics.high_rhat_count)")
+        println("  Min ESS: $(round(diagnostics.min_ess, digits=0))")
+        println("  Mean ESS: $(round(diagnostics.mean_ess, digits=0))")
+        println("  Convergence Status: $(diagnostics.convergence_status)")
+
+        # Step 7: Save results
+        println("\n" * "="^50)
+        println("STEP 7: SAVING RESULTS")
+        println("="^50)
+        println("  Using save-local-then-move strategy...")
+
+        # Prepare data for saving (SINGLE copy of stanmodel, not multiple!)
+        model_data_to_save = Dict{String, Any}(
+            # StanModel object (contains all MCMC samples)
+            "stanmodel_object" => stanmodel,
+
+            # Processed diagnostics
+            "diagnostics_summary" => diagnostics.diagnostics_summary,
+            "data_dict" => dat_4dim,
+
+            # Original data for reference
+            "manifesto" => final_data.manifesto,
+            "expert_dim" => final_data.expert_dim,
+            "expert_lr" => final_data.expert_lr,
+            "segment_year" => final_data.segment_year,  # V10: segment-year mapping
+            "segment_info" => final_data.segment_info,  # V10: segment metadata (party_id, segment_num, year range)
+
+            # Metadata with convergence info
+            "model_info" => Dict(
+                "timestamp" => Dates.format(Dates.now(), "yyyy-mm-dd_HH-MM-SS"),
+                "max_rhat" => diagnostics.max_rhat,
+                "mean_rhat" => diagnostics.mean_rhat,
+                "min_ess" => diagnostics.min_ess,
+                "mean_ess" => diagnostics.mean_ess,
+                "convergence_status" => diagnostics.convergence_status,
+                "model_file" => model_file,
+                "model_version" => MODEL_VERSION,
+                "num_chains" => num_chains,
+                "num_warmup" => num_warmup,
+                "num_samples" => num_samples,
+                "adapt_delta" => adapt_delta,
+                "max_depth" => max_depth,
+                "year0" => year0,
+                "dimensions" => MODEL_VERSION == "2dim" ?
+                    ["economic_lr", "galtan"] :
+                    ["pro_market", "pro_welfare", "cosmopolitan", "traditional"]
+            )
+        )
+
+        # Save using CSV-first robust system. Chains have already been secured
+        # by model execution; robust_save_model adds metadata/data and verifies.
+        save_dir = data_dir != "data" ? joinpath(data_dir, "model_run") : "outputs/model_outputs"
+        output_file = robust_save_model(
+            stanmodel,
+            model_data_to_save,
+            save_dir;
+            compress=true,       # Ignored by CSV-first save implementation
+            keep_local_backups=2  # Ignored; chains are already saved before this step
+        )
+
+        # Robust save module already verified everything!
+
+        println("\n" * "="^70)
+        println("MODEL EXECUTION COMPLETED SUCCESSFULLY!")
+        println("="^70)
+        println("   Max R-hat: $(round(diagnostics.max_rhat, digits=4))")
+        println("   Mean R-hat: $(round(diagnostics.mean_rhat, digits=4))")
+        println("   Convergence: $(diagnostics.convergence_status)")
+        println("   Output file: $output_file")
+
+        # Print summary statistics
+        println("\nModel Summary ($(MODEL_VERSION == "2dim" ? "2D Direct Bipolar" : "4D Unipolar")):")
+        println("  Segments: $(data_prep.S)")
+        println("  Parties with valid segments: $(data_prep.J)")
+        println("  Countries: $(data_prep.P)")
+        println("  Segment-year combinations: $(data_prep.R)")
+        println("  Years: $(final_data.T_year)")
+        println("  Manifesto observations: $(dat_4dim["N_man"])")
+        println("  Expert dimension-specific observations: $(dat_4dim["N_exp_dim"])")
+        println("  Expert general L-R observations: $(dat_4dim["N_exp_lr"])")
+        if MODEL_VERSION == "2dim"
+            println("  Dimensions estimated: 2 (economic_lr, galtan)")
+            println("  Theta parameters: $(2 * data_prep.R) (2 × R)")
+        else
+            println("  Dimensions estimated: 4 (pro_market, pro_welfare, cosmopolitan, traditional)")
+            println("  Theta parameters: $(4 * data_prep.R) (4 × R)")
+        end
+        println("=" ^ 70)
+
+        # Cleanup temp folder after successful save
+        println("\nCLEANING UP TEMPORARY FILES...")
+        try
+            if isdir(temp_folder)
+                rm(temp_folder, recursive=true, force=true)
+                println("  Removed temporary folder: $temp_folder")
+            end
+        catch cleanup_error
+            println("  Warning: Could not remove temp folder: $cleanup_error")
+            println("  (This won't affect your saved results)")
+        end
+
+        return true
+
+    catch e
+        println("\nERROR in model pipeline: $e")
+        println("Stack trace:")
+        showerror(stdout, e, catch_backtrace())
+        rethrow(e)
+    end
+end
+
+function main(args=ARGS)
+    # Parse --data-dir argument
+    data_dir = "data"
+    for (i, arg) in enumerate(args)
+        if arg == "--data-dir" && i < length(args)
+            data_dir = args[i + 1]
+        elseif startswith(arg, "--data-dir=")
+            data_dir = split(arg, "=", limit=2)[2]
+        end
+    end
+
+    if data_dir != "."
+        println("Using data directory: $data_dir")
+    end
+    println("Executing $(MODEL_VERSION) latent trait model pipeline...")
+
+    # Check that required data files exist (in data_dir)
+    required_data = [joinpath(data_dir, f) for f in ["text_data.csv", "expert.csv", "lr_data.csv"]]
+    required_files = vcat(required_data, [STAN_MODEL_FILE])
+    missing_files = []
+
+    for file in required_files
+        if !isfile(file)
+            push!(missing_files, file)
+        end
+    end
+
+    if !isempty(missing_files)
+        println("ERROR: Missing required files:")
+        for file in missing_files
+            println("  - $file")
+        end
+
+        if any(f -> endswith(f, "text_data.csv") || endswith(f, "expert.csv") || endswith(f, "lr_data.csv"), missing_files)
+            println("\nTo generate data files, run:")
+            println("  bash scripts/01_prepare_data.sh")
+        end
+
+        error("Cannot proceed without required files")
+    end
+
+    # Run the complete pipeline
+    results = run_model(data_dir=data_dir)
+
+    println("\n$(MODEL_VERSION) latent trait model pipeline completed successfully!")
+    println("Check outputs/model_outputs/latest/ for chain CSVs and metadata.")
+end
+
+# Main execution
+if abspath(PROGRAM_FILE) == @__FILE__
+    main()
+end

src/julia/02_enrich_output.jl

@@ -0,0 +1,211 @@
+#!/usr/bin/env julia
+#=
+02_enrich_output.jl - Enrich party positions CSV with metadata
+
+Fast post-processing script that adds party names, union membership status,
+and election results to the model output CSV. Operates purely on CSV files
+(no chain loading). Takes seconds, not minutes.
+
+Usage:
+    julia 02_enrich_output.jl                # enriches latest party_positions_*.csv
+    julia 02_enrich_output.jl somefile.csv    # enriches a specific file
+
+Adds columns:
+    party_name_english  - Full English party name (from data/party_names.csv)
+    party_name_short    - Standard abbreviation
+    in_union            - 1 if party's union had a joint manifesto that year, 0 otherwise
+    pervote             - Vote share (%) at election years; missing for non-election years
+=#
+
+using CSV
+using DataFrames
+using Dates
+
+function find_latest_output()
+    outdir = "outputs/estimations/latest"
+    files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
+                        !contains(f, "metadata") && !contains(f, "tables"),
+                   readdir(outdir))
+    isempty(files) && error("No party_positions_*.csv found in $outdir/")
+    sort!(files, rev=true)
+    return joinpath(outdir, files[1])
+end
+
+function enrich(input_file::String)
+    println("="^60)
+    println("ENRICH OUTPUT")
+    println("="^60)
+    println("Input: $input_file")
+
+    output = CSV.read(input_file, DataFrame)
+    println("  Rows: $(nrow(output)), Columns: $(ncol(output))")
+
+    # --- Party names ---
+    party_names_file = joinpath("data", "party_names.csv")
+    if isfile(party_names_file)
+        names_df = CSV.read(party_names_file, DataFrame)
+        name_lookup = Dict{Int, Tuple{String, String}}()
+        for row in eachrow(names_df)
+            short = ismissing(row.party_name_short) ? "" : string(row.party_name_short)
+            name_lookup[row.partyfacts_id] = (string(row.party_name_english), short)
+        end
+        output.party_name_english = [
+            haskey(name_lookup, pid) ? name_lookup[pid][1] : ""
+            for pid in output.party_id
+        ]
+        output.party_name_short = [
+            haskey(name_lookup, pid) ? name_lookup[pid][2] : ""
+            for pid in output.party_id
+        ]
+        n_named = count(x -> x != "", output.party_name_english)
+        println("  Party names: $n_named / $(nrow(output)) rows matched")
+    else
+        output.party_name_english = fill("", nrow(output))
+        output.party_name_short = fill("", nrow(output))
+        println("  WARNING: data/party_names.csv not found")
+    end
+
+    # --- Union mapping (for in_union + pervote fallback) ---
+    union_mapping_file = joinpath("data", "union_mapping.csv")
+    constituent_to_union = Dict{Int, Int}()
+    if isfile(union_mapping_file)
+        union_df = CSV.read(union_mapping_file, DataFrame)
+        for row in eachrow(union_df)
+            constituent_to_union[row.expert_pf_id] = row.manifesto_pf_id
+        end
+    end
+
+    # --- in_union dummy (year-varying) ---
+    text_data_file = "data/text_data.csv"
+    output.in_union = zeros(Int, nrow(output))
+
+    if isfile(text_data_file) && !isempty(constituent_to_union)
+        text_df = CSV.read(text_data_file, DataFrame)
+
+        # Build set of (party_pf_id, year) pairs for manifesto data
+        manifesto_text = filter(r -> r.project == "Manifesto Project", text_df)
+        manifesto_party_years = Set{Tuple{Int, Int}}()
+        for row in eachrow(manifesto_text)
+            push!(manifesto_party_years, (row.party, row.year))
+        end
+
+        # Process each (party, segment) group
+        gdf = groupby(output, [:party_id, :segment_num])
+        for subdf in gdf
+            pid = subdf.party_id[1]
+            !haskey(constituent_to_union, pid) && continue
+            union_id = constituent_to_union[pid]
+
+            # Get row indices in the full output for this group
+            idxs = parentindices(subdf)[1]
+
+            # Determine in_union at election years
+            election_year_vals = Dict{Int, Int}()
+            for (j, row) in enumerate(eachrow(subdf))
+                if (union_id, row.year) in manifesto_party_years
+                    election_year_vals[row.year] = 1
+                elseif (pid, row.year) in manifesto_party_years
+                    election_year_vals[row.year] = 0
+                end
+            end
+
+            # Forward-fill within segment
+            sorted_pairs = sort(collect(zip(subdf.year, idxs)))
+            last_val = 0
+            for (yr, idx) in sorted_pairs
+                if haskey(election_year_vals, yr)
+                    last_val = election_year_vals[yr]
+                end
+                output.in_union[idx] = last_val
+            end
+        end
+
+        n_in_union = count(x -> x == 1, output.in_union)
+        println("  in_union: $n_in_union rows flagged as union members")
+    else
+        println("  WARNING: Could not compute in_union (missing files)")
+    end
+
+    # --- Election results (pervote) ---
+    election_file = joinpath("data", "election_data.csv")
+    output.pervote = Vector{Union{Float64, Missing}}(missing, nrow(output))
+
+    if isfile(election_file)
+        election_df = CSV.read(election_file, DataFrame)
+
+        # Build lookup: (party_id, year) -> pervote
+        election_lookup = Dict{Tuple{Int, Int}, Float64}()
+        for row in eachrow(election_df)
+            election_lookup[(row.party, row.year)] = row.pervote
+        end
+
+        n_filled = 0
+        for i in 1:nrow(output)
+            pid = output.party_id[i]
+            yr = output.year[i]
+
+            # Direct match (standalone party)
+            if haskey(election_lookup, (pid, yr))
+                output.pervote[i] = election_lookup[(pid, yr)]
+                n_filled += 1
+            elseif hasproperty(output, :union_party_id) && !ismissing(output.union_party_id[i])
+                # Union constituent: look up by union PF ID
+                uid = output.union_party_id[i]
+                if haskey(election_lookup, (uid, yr))
+                    output.pervote[i] = election_lookup[(uid, yr)]
+                    n_filled += 1
+                end
+            elseif haskey(constituent_to_union, pid)
+                # Fallback: use union mapping even if union_party_id column not populated
+                uid = constituent_to_union[pid]
+                if haskey(election_lookup, (uid, yr))
+                    output.pervote[i] = election_lookup[(uid, yr)]
+                    n_filled += 1
+                end
+            end
+        end
+        println("  pervote: $n_filled / $(nrow(output)) rows filled")
+    else
+        println("  WARNING: election_data.csv not found")
+    end
+
+    # --- Reorder columns ---
+    estimate_cols = Symbol[]
+    for base in ["economic_lr", "galtan", "pro_market", "pro_welfare", "cosmopolitan", "traditional"]
+        sym = Symbol(base)
+        if hasproperty(output, sym)
+            push!(estimate_cols, sym)
+            push!(estimate_cols, Symbol("$(base)_se"))
+            push!(estimate_cols, Symbol("$(base)_q025"))
+            push!(estimate_cols, Symbol("$(base)_q975"))
+        end
+    end
+
+    # Fix country code: MO (Macau) → MK (North Macedonia) — GPS uses wrong ISO2
+    output.country = replace(output.country, "MO" => "MK")
+
+    col_order = vcat(
+        [:party_id, :party_name_english, :party_name_short, :country, :year, :segment_num,
+         :union_party_id, :in_union, :pervote],
+        estimate_cols
+    )
+    col_order = filter(c -> hasproperty(output, c), col_order)
+    select!(output, col_order)
+
+    # --- Write back ---
+    CSV.write(input_file, output)
+    println("\n  Wrote: $input_file")
+    println("  Columns ($(ncol(output))): $(join(string.(names(output)), ", "))")
+
+    return output
+end
+
+function main(args=ARGS)
+    input = length(args) >= 1 ? args[1] : find_latest_output()
+    enrich(input)
+    println("\nDone.")
+end
+
+if abspath(PROGRAM_FILE) == @__FILE__
+    main()
+end

src/julia/02_post_estimation.jl

@@ -0,0 +1,985 @@
+#!/usr/bin/env julia
+#=
+02_post_estimation.jl - Extract party position estimates from Stan model output
+
+Supports both model versions:
+- 2D model (V1): Extracts economic_lr and galtan directly
+- 4D model (V10): Extracts 4 traits + 2 derived scales
+
+V10/V1 UPDATE: Handles segment-based indexing
+- Maps segment results back to original party IDs
+- Adds segment_num column to indicate which segment of the party
+- Flags discontinuities for parties with multiple segments
+
+This script:
+1. Auto-detects the latest model run in model_outputs/
+2. Loads the chain CSV files and data mappings
+3. Detects model version from metadata or column names
+4. Extracts posterior summaries for all segment-year positions
+5. Maps Stan parameter indices back to real party IDs, segment numbers, and years
+6. Saves output as wide-format CSV with uncertainty estimates
+
+Usage:
+    julia 02_post_estimation.jl
+
+Output:
+    party_positions_YYYY-MM-DD_HH-MM-SS.csv
+=#
+
+using CSV
+using DataFrames
+using Statistics
+using JSON
+using Dates
+using Printf
+
+# =============================================================================
+# STEP 0: Auto-detect latest run
+# =============================================================================
+
+function find_latest_run(base_dir::String="outputs/model_outputs/latest")
+    if !isdir(base_dir)
+        error("Model outputs directory not found: $base_dir")
+    end
+
+    runs = filter(d -> startswith(d, "run_") && isdir(joinpath(base_dir, d)), readdir(base_dir))
+
+    if isempty(runs)
+        error("No runs found in $base_dir")
+    end
+
+    # Sort by timestamp in directory name (format: run_YYYY-MM-DD_HH-MM-SS)
+    sort!(runs, rev=true)
+
+    latest = joinpath(base_dir, runs[1])
+    println("Found $(length(runs)) run(s). Using latest: $latest")
+    return latest
+end
+
+# =============================================================================
+# STEP 1: Load data and build segment-year lookup
+# =============================================================================
+
+function load_run_data(run_dir::String)
+    println("\n" * "="^60)
+    println("LOADING RUN DATA")
+    println("="^60)
+
+    data_dir = joinpath(run_dir, "data")
+    chains_dir = joinpath(run_dir, "chains")
+
+    # Check required files exist
+    required_files = [
+        joinpath(data_dir, "text_data.csv"),
+        joinpath(data_dir, "expert_dim.csv"),
+        joinpath(data_dir, "expert_lr.csv"),
+        joinpath(run_dir, "metadata.json")
+    ]
+
+    for f in required_files
+        if !isfile(f)
+            error("Required file not found: $f")
+        end
+    end
+
+    # Load data files
+    println("Loading text_data.csv...")
+    text_data = CSV.read(joinpath(data_dir, "text_data.csv"), DataFrame)
+    println("  Rows: $(nrow(text_data))")
+
+    println("Loading expert_dim.csv...")
+    expert_dim = CSV.read(joinpath(data_dir, "expert_dim.csv"), DataFrame)
+    println("  Rows: $(nrow(expert_dim))")
+
+    println("Loading expert_lr.csv...")
+    expert_lr = CSV.read(joinpath(data_dir, "expert_lr.csv"), DataFrame)
+    println("  Rows: $(nrow(expert_lr))")
+
+    println("Loading metadata.json...")
+    metadata = JSON.parsefile(joinpath(run_dir, "metadata.json"))
+    println("  year0: $(metadata["year0"])")
+    println("  Model: $(metadata["model_file"])")
+
+    # V10: Load segment_info if available
+    segment_info_file = joinpath(data_dir, "segment_info.csv")
+    segment_info = nothing
+    if isfile(segment_info_file)
+        println("Loading segment_info.csv (V10)...")
+        segment_info = CSV.read(segment_info_file, DataFrame)
+        println("  Segments: $(nrow(segment_info))")
+    end
+
+    # V10: Load segment_year_map if available
+    segment_year_file = joinpath(data_dir, "segment_year_map.csv")
+    segment_year_map = nothing
+    if isfile(segment_year_file)
+        println("Loading segment_year_map.csv (V10)...")
+        segment_year_map = CSV.read(segment_year_file, DataFrame)
+        println("  Segment-years: $(nrow(segment_year_map))")
+    end
+
+    # Find chain files
+    chain_files = filter(f -> endswith(f, ".csv") && startswith(f, "chain_"), readdir(chains_dir))
+    println("\nFound $(length(chain_files)) chain file(s)")
+
+    return (
+        text_data = text_data,
+        expert_dim = expert_dim,
+        expert_lr = expert_lr,
+        metadata = metadata,
+        segment_info = segment_info,
+        segment_year_map = segment_year_map,
+        chain_files = [joinpath(chains_dir, f) for f in sort(chain_files)],
+        run_dir = run_dir
+    )
+end
+
+function normalize_country_value(value)
+    if ismissing(value)
+        return missing
+    end
+    txt = strip(string(value))
+    return isempty(txt) ? missing : txt
+end
+
+function build_party_country_map(text_data::DataFrame, expert_dim::DataFrame, expert_lr::DataFrame)
+    merged = unique(vcat(
+        select(text_data, :party, :country),
+        select(expert_dim, :party, :country),
+        select(expert_lr, :party, :country)
+    ))
+
+    party_to_country = Dict{Int, String}()
+    for row in eachrow(merged)
+        pid = tryparse(Int, string(row.party))
+        if pid === nothing
+            continue
+        end
+        c = normalize_country_value(row.country)
+        if !ismissing(c)
+            party_to_country[pid] = c
+        end
+    end
+    return party_to_country
+end
+
+function load_constituent_to_union_map()::Dict{Int, Int}
+    mapping_file = joinpath("data", "union_mapping.csv")
+    constituent_to_union = Dict{Int, Int}()
+    if isfile(mapping_file)
+        union_df = CSV.read(mapping_file, DataFrame)
+        for row in eachrow(union_df)
+            constituent_to_union[row.expert_pf_id] = row.manifesto_pf_id
+        end
+    end
+    return constituent_to_union
+end
+
+function resolve_party_country(pid_value,
+                               party_to_country::Dict{Int, String},
+                               constituent_to_union::Dict{Int, Int})
+    pid = tryparse(Int, string(pid_value))
+    if pid === nothing
+        return missing, "unresolved"
+    end
+
+    if haskey(party_to_country, pid)
+        return party_to_country[pid], "direct"
+    end
+
+    if haskey(constituent_to_union, pid)
+        uid = constituent_to_union[pid]
+        if haskey(party_to_country, uid)
+            return party_to_country[uid], "union_fallback"
+        end
+    end
+
+    return missing, "unresolved"
+end
+
+function apply_country_resolution!(df::DataFrame,
+                                   party_col::Symbol,
+                                   country_col::Symbol,
+                                   party_to_country::Dict{Int, String},
+                                   constituent_to_union::Dict{Int, Int})
+    resolved_country = Union{Missing, String}[]
+    source_counts = Dict("direct" => 0, "union_fallback" => 0, "unresolved" => 0)
+    unresolved_parties = Set{Int}()
+
+    for pid in df[!, party_col]
+        country, source = resolve_party_country(pid, party_to_country, constituent_to_union)
+        push!(resolved_country, country)
+        source_counts[source] += 1
+        if source == "unresolved"
+            pid_int = tryparse(Int, string(pid))
+            if pid_int !== nothing
+                push!(unresolved_parties, pid_int)
+            end
+        end
+    end
+
+    df[!, country_col] = resolved_country
+    return source_counts, sort!(collect(unresolved_parties))
+end
+
+function fill_missing_countries!(df::DataFrame,
+                                 segment_info::Union{DataFrame, Nothing},
+                                 party_to_country::Dict{Int, String},
+                                 constituent_to_union::Dict{Int, Int})
+    if !hasproperty(df, :country)
+        source_counts, unresolved = apply_country_resolution!(
+            df, :party_id, :country, party_to_country, constituent_to_union
+        )
+        return source_counts, unresolved
+    end
+
+    normalized = Union{Missing, String}[]
+    for val in df.country
+        push!(normalized, normalize_country_value(val))
+    end
+    df.country = normalized
+
+    source_counts = Dict("direct" => 0, "union_fallback" => 0, "segment_info" => 0, "unresolved" => 0)
+
+    segment_country_by_id = Dict{Int, String}()
+    if segment_info !== nothing && hasproperty(segment_info, :country)
+        for row in eachrow(segment_info)
+            c = normalize_country_value(row.country)
+            if !ismissing(c)
+                segment_country_by_id[Int(row.segment_id)] = c
+            end
+        end
+    end
+
+    unresolved_parties = Set{Int}()
+    for i in 1:nrow(df)
+        if !ismissing(df.country[i])
+            continue
+        end
+
+        if hasproperty(df, :segment_id) && haskey(segment_country_by_id, Int(df.segment_id[i]))
+            df.country[i] = segment_country_by_id[Int(df.segment_id[i])]
+            source_counts["segment_info"] += 1
+            continue
+        end
+
+        country, source = resolve_party_country(df.party_id[i], party_to_country, constituent_to_union)
+        if !ismissing(country)
+            df.country[i] = country
+            source_counts[source] += 1
+        else
+            source_counts["unresolved"] += 1
+            pid_int = tryparse(Int, string(df.party_id[i]))
+            pid_int !== nothing && push!(unresolved_parties, pid_int)
+        end
+    end
+
+    return source_counts, sort!(collect(unresolved_parties))
+end
+
+# =============================================================================
+# STEP 2: Build complete segment-year mapping (V10) or party-year mapping (V9)
+# =============================================================================
+
+function build_segment_year_map(text_data::DataFrame, expert_dim::DataFrame, expert_lr::DataFrame,
+                                 segment_info::Union{DataFrame, Nothing},
+                                 segment_year_map::Union{DataFrame, Nothing},
+                                 run_dir::String,
+                                 year0::Int)
+    println("\n" * "="^60)
+    println("BUILDING SEGMENT-YEAR MAPPING")
+    println("="^60)
+
+    party_to_country = build_party_country_map(text_data, expert_dim, expert_lr)
+    constituent_to_union = load_constituent_to_union_map()
+
+    # V10: Use segment_year_map if available
+    if segment_year_map !== nothing && segment_info !== nothing
+        println("Using segment_year_map.csv (V10 mode)")
+
+        # Convert relative Year to absolute year
+        if hasproperty(segment_year_map, :Year)
+            segment_year_map.year = segment_year_map.Year .+ year0
+        elseif !hasproperty(segment_year_map, :year)
+            error("segment_year_map has no Year or year column")
+        end
+
+        # Add party_id from segment_info if not already present
+        if !hasproperty(segment_year_map, :party_id)
+            segment_id_to_party = Dict(row.segment_id => row.party_id for row in eachrow(segment_info))
+            segment_year_map.party_id = [segment_id_to_party[sid] for sid in segment_year_map.segment_id]
+        end
+
+        # Add segment_num from segment_info if not already present
+        if !hasproperty(segment_year_map, :segment_num)
+            segment_id_to_segnum = Dict(row.segment_id => row.segment_num for row in eachrow(segment_info))
+            segment_year_map.segment_num = [segment_id_to_segnum[sid] for sid in segment_year_map.segment_id]
+        end
+
+        # Resolve/fill country column using segment metadata first, then direct and union-fallback lookup.
+        source_counts, unresolved = fill_missing_countries!(
+            segment_year_map, segment_info, party_to_country, constituent_to_union
+        )
+        direct_count = get(source_counts, "direct", 0)
+        union_count = get(source_counts, "union_fallback", 0)
+        segment_info_count = get(source_counts, "segment_info", 0)
+        unresolved_count = count(ismissing, segment_year_map.country)
+        println("  Country resolution fill counts: direct=$direct_count, union_fallback=$union_count, segment_info=$segment_info_count, unresolved_rows=$unresolved_count")
+        if !isempty(unresolved)
+            println("  Warning: unresolved country party IDs (first 20): $(unresolved[1:min(20, length(unresolved))])")
+        end
+
+        R = maximum(segment_year_map.rr)
+        n_segments = length(unique(segment_year_map.segment_id))
+        n_parties = length(unique(segment_year_map.party_id))
+
+        println("Loaded segment_year_map: $(nrow(segment_year_map)) segment-years (R=$R)")
+        println("  Unique segments: $n_segments")
+        println("  Unique parties: $n_parties")
+
+        # Count observed vs interpolated
+        observed_rrs = Set{Int}()
+        if hasproperty(text_data, :rr_man)
+            union!(observed_rrs, Set(text_data.rr_man))
+        end
+        if hasproperty(expert_dim, :rr_exp_dim)
+            union!(observed_rrs, Set(expert_dim.rr_exp_dim))
+        end
+        if hasproperty(expert_lr, :rr_exp_lr)
+            union!(observed_rrs, Set(expert_lr.rr_exp_lr))
+        end
+
+        n_observed = length(intersect(Set(segment_year_map.rr), observed_rrs))
+        n_interpolated = nrow(segment_year_map) - n_observed
+        println("  Observed segment-years: $n_observed")
+        println("  Interpolated segment-years: $n_interpolated")
+
+        return segment_year_map, R, segment_info
+    end
+
+    # V9 fallback: Use party_year_map
+    party_year_file = joinpath(run_dir, "data", "party_year_map.csv")
+    if isfile(party_year_file)
+        println("Loading party_year_map.csv (V9 fallback mode)")
+        party_year_map = CSV.read(party_year_file, DataFrame)
+
+        # Add party_id column (same as party for V9)
+        if !hasproperty(party_year_map, :party_id)
+            party_year_map.party_id = party_year_map.party
+        end
+
+        # Add segment_num column (always 1 for V9)
+        if !hasproperty(party_year_map, :segment_num)
+            party_year_map.segment_num = ones(Int, nrow(party_year_map))
+        end
+
+        # Add segment_id column (same as party index for V9)
+        if !hasproperty(party_year_map, :segment_id)
+            party_year_map.segment_id = party_year_map.party
+        end
+
+        # Convert relative Year to absolute year
+        if hasproperty(party_year_map, :Year)
+            party_year_map.year = party_year_map.Year .+ year0
+        elseif !hasproperty(party_year_map, :year)
+            error("party_year_map has no Year or year column")
+        end
+
+        # Resolve/fill country column
+        if !hasproperty(party_year_map, :country)
+            source_counts, unresolved = apply_country_resolution!(
+                party_year_map, :party_id, :country, party_to_country, constituent_to_union
+            )
+            direct_count = source_counts["direct"]
+            union_count = source_counts["union_fallback"]
+            unresolved_count = source_counts["unresolved"]
+            println("  Country resolution sources: direct=$direct_count, union_fallback=$union_count, unresolved=$unresolved_count")
+            if !isempty(unresolved)
+                println("  Warning: unresolved country party IDs (first 20): $(unresolved[1:min(20, length(unresolved))])")
+            end
+        else
+            normalized = Union{Missing, String}[]
+            for val in party_year_map.country
+                push!(normalized, normalize_country_value(val))
+            end
+            party_year_map.country = normalized
+        end
+
+        R = maximum(party_year_map.rr)
+        println("Loaded party_year_map: $(nrow(party_year_map)) party-years (R=$R)")
+
+        return party_year_map, R, nothing
+    end
+
+    # Fallback: Reconstruct from data files
+    @warn "No mapping file found, reconstructing from data (observed years only)"
+
+    # Extract unique party-year-rr combinations from text_data
+    text_map = unique(select(text_data, :party, :country, :year, :rr_man))
+    rename!(text_map, :rr_man => :rr)
+    text_map.party_id = text_map.party
+    text_map.segment_num = ones(Int, nrow(text_map))
+
+    expert_dim_map = unique(select(expert_dim, :party, :country, :year, :rr_exp_dim))
+    rename!(expert_dim_map, :rr_exp_dim => :rr)
+    expert_dim_map.party_id = expert_dim_map.party
+    expert_dim_map.segment_num = ones(Int, nrow(expert_dim_map))
+
+    expert_lr_map = unique(select(expert_lr, :party, :country, :year, :rr_exp_lr))
+    rename!(expert_lr_map, :rr_exp_lr => :rr)
+    expert_lr_map.party_id = expert_lr_map.party
+    expert_lr_map.segment_num = ones(Int, nrow(expert_lr_map))
+
+    combined = vcat(text_map, expert_dim_map, expert_lr_map)
+    segment_year_map = unique(combined)
+    sort!(segment_year_map, :rr)
+
+    R = maximum(segment_year_map.rr)
+    println("Reconstructed mapping: $(nrow(segment_year_map)) segment-years (R=$R)")
+
+    return segment_year_map, R, nothing
+end
+
+# =============================================================================
+# STEP 3: Load and combine chains
+# =============================================================================
+
+function load_chains(chain_files::Vector{String})
+    println("\n" * "="^60)
+    println("LOADING STAN CHAINS")
+    println("="^60)
+    flush(stdout)
+
+    chains = DataFrame[]
+
+    # The full Stan CSVs are very wide (hundreds of thousands of columns).  For
+    # post-estimation we only need party-position generated quantities.  Reading
+    # all columns can take hours and allocate many GB of irrelevant parameters.
+    post_estimation_prefixes = (
+        "economic_lr.",
+        "galtan.",
+        "pro_market.",
+        "pro_welfare.",
+        "cosmopolitan.",
+        "traditional.",
+    )
+    keep_post_estimation_col(_i, name) = any(startswith(String(name), p) for p in post_estimation_prefixes)
+
+    for (i, f) in enumerate(chain_files)
+        println("Loading chain $i: $(basename(f))...")
+        flush(stdout)
+        # Skip comment lines (Stan header) and parse only needed quantities.
+        chain = CSV.read(f, DataFrame; comment="#", select=keep_post_estimation_col)
+        println("  Samples: $(nrow(chain)), Parameters: $(ncol(chain))")
+        flush(stdout)
+        push!(chains, chain)
+    end
+
+    # Combine chains
+    println("Combining selected chain columns...")
+    flush(stdout)
+    combined = vcat(chains...)
+    println("\nCombined: $(nrow(combined)) total samples")
+    println("Selected parameters: $(ncol(combined))")
+    flush(stdout)
+
+    return combined
+end
+
+# =============================================================================
+# STEP 4: Extract generated quantities
+# =============================================================================
+
+"""
+Detect model version from chain column names.
+Returns "2dim" or "4dim".
+"""
+function detect_model_version(chains::DataFrame)
+    cols = names(chains)
+    # 2D model has economic_lr but NOT pro_market
+    has_economic_lr = any(c -> startswith(string(c), "economic_lr."), cols)
+    has_pro_market = any(c -> startswith(string(c), "pro_market."), cols)
+
+    if has_economic_lr && !has_pro_market
+        return "2dim"
+    elseif has_pro_market
+        return "4dim"
+    else
+        error("Could not detect model version from chain columns")
+    end
+end
+
+function extract_estimates(chains::DataFrame, segment_year_map::DataFrame, R::Int)
+    println("\n" * "="^60)
+    println("EXTRACTING POSTERIOR ESTIMATES")
+    println("="^60)
+
+    # Auto-detect model version from columns
+    model_version = detect_model_version(chains)
+    println("Detected model version: $model_version")
+
+    # Select quantities based on model version
+    if model_version == "2dim"
+        # 2D model: economic_lr and galtan are directly estimated
+        # (general_lr is computed in Stan for anchoring but not extracted as output)
+        quantities = ["economic_lr", "galtan"]
+        test_col = "economic_lr.1"
+    else
+        # 4D model: 4 traits + 2 derived scales
+        quantities = ["pro_market", "pro_welfare", "cosmopolitan", "traditional", "economic_lr", "galtan"]
+        test_col = "pro_market.1"
+    end
+
+    # Check that columns exist
+    if !hasproperty(chains, Symbol(test_col))
+        error("Column $test_col not found in chains. Available columns: $(first(names(chains), 10))...")
+    end
+
+    n_samples = nrow(chains)
+    println("Samples per parameter: $n_samples")
+
+    # Load union mapping for adding union_party_id column
+    union_mapping_file = joinpath("data", "union_mapping.csv")
+    constituent_to_union_pf = Dict{Int, Int}()
+    if isfile(union_mapping_file)
+        union_df = CSV.read(union_mapping_file, DataFrame)
+        for row in eachrow(union_df)
+            constituent_to_union_pf[row.expert_pf_id] = row.manifesto_pf_id
+        end
+    end
+
+    # Pre-allocate output DataFrame
+    n_rows = nrow(segment_year_map)
+
+    # Add union_party_id column: NA for standalone parties, union PF ID for constituents
+    union_ids = Union{Int, Missing}[]
+    for pid in segment_year_map.party_id
+        pid_int = isa(pid, Integer) ? pid : tryparse(Int, string(pid))
+        if pid_int !== nothing && haskey(constituent_to_union_pf, pid_int)
+            push!(union_ids, constituent_to_union_pf[pid_int])
+        else
+            push!(union_ids, missing)
+        end
+    end
+
+    output = DataFrame(
+        party_id = segment_year_map.party_id,
+        union_party_id = union_ids,
+        segment_num = segment_year_map.segment_num,
+        country = segment_year_map.country,
+        year = segment_year_map.year,
+        rr = segment_year_map.rr
+    )
+
+    # Add columns for each quantity
+    for q in quantities
+        output[!, Symbol(q)] = zeros(Float64, n_rows)
+        output[!, Symbol("$(q)_se")] = zeros(Float64, n_rows)
+        output[!, Symbol("$(q)_q025")] = zeros(Float64, n_rows)
+        output[!, Symbol("$(q)_q975")] = zeros(Float64, n_rows)
+    end
+
+    println("Extracting estimates for $(n_rows) segment-year positions...")
+
+    # Progress tracking
+    prog_interval = max(1, n_rows ÷ 20)
+
+    for (i, row) in enumerate(eachrow(segment_year_map))
+        r = row.rr
+
+        # Progress
+        if i % prog_interval == 0 || i == n_rows
+            pct = round(100 * i / n_rows, digits=1)
+            print("\r  Progress: $pct% ($i / $n_rows)")
+        end
+
+        for q in quantities
+            col_name = Symbol("$q.$r")
+
+            if !hasproperty(chains, col_name)
+                @warn "Column $col_name not found (rr=$r)" maxlog=5
+                continue
+            end
+
+            samples = chains[!, col_name]
+
+            # Compute summary statistics
+            output[i, Symbol(q)] = mean(samples)
+            output[i, Symbol("$(q)_se")] = std(samples)
+            output[i, Symbol("$(q)_q025")] = quantile(samples, 0.025)
+            output[i, Symbol("$(q)_q975")] = quantile(samples, 0.975)
+        end
+    end
+    println()  # Newline after progress
+
+    # Remove the rr column from final output (internal only)
+    select!(output, Not(:rr))
+
+    return output
+end
+
+# =============================================================================
+# STEP 5: Validation
+# =============================================================================
+
+function validate_output(output::DataFrame, segment_info::Union{DataFrame, Nothing})
+    println("\n" * "="^60)
+    println("VALIDATION CHECKS")
+    println("="^60)
+
+    all_passed = true
+
+    # Detect which columns are present (2D vs 4D model)
+    has_4d = hasproperty(output, :pro_market)
+
+    # Check 1: Range check - all estimates should be in [0, 1]
+    println("\n1. Range check (all values in [0, 1]):")
+
+    if has_4d
+        check_cols = [:pro_market, :pro_welfare, :cosmopolitan, :traditional, :economic_lr, :galtan]
+    else
+        check_cols = [:economic_lr, :galtan]
+    end
+
+    for col in check_cols
+        if !hasproperty(output, col)
+            continue
+        end
+        vals = output[!, col]
+        min_val, max_val = extrema(vals)
+        in_range = min_val >= 0 && max_val <= 1
+        status = in_range ? "PASS" : "FAIL"
+        println("   $col: [$(@sprintf("%.4f", min_val)), $(@sprintf("%.4f", max_val))] - $status")
+        all_passed = all_passed && in_range
+    end
+
+    # Check 2: Anchor party checks
+    println("\n2. Anchor party checks:")
+
+    # Define anchor parties with expected ranges (for 2D model)
+    # Includes both union IDs (V3) and individual constituent IDs (V4)
+    anchor_parties = [
+        (id=211, name="CDU/CSU", country="DE", econ=(0.50, 0.70), galtan=(0.45, 0.70)),
+        (id=1375, name="CDU", country="DE", econ=(0.50, 0.70), galtan=(0.45, 0.65)),
+        (id=1731, name="CSU", country="DE", econ=(0.50, 0.70), galtan=(0.55, 0.75)),
+        (id=383, name="SPD", country="DE", econ=(0.30, 0.50), galtan=(0.30, 0.55)),
+        (id=1516, name="Labour", country="GB", econ=(0.30, 0.55), galtan=(0.30, 0.55)),
+        (id=1567, name="Conservatives", country="GB", econ=(0.55, 0.80), galtan=(0.50, 0.75)),
+        (id=487, name="SAP", country="SE", econ=(0.30, 0.50), galtan=(0.35, 0.55)),
+    ]
+
+    n_checked = 0
+    n_passed = 0
+
+    for anchor in anchor_parties
+        party_rows = filter(r -> r.party_id == anchor.id, output)
+
+        if nrow(party_rows) == 0
+            println("   $(anchor.name) ($(anchor.id)): NOT FOUND")
+            continue
+        end
+
+        # Use most recent 20 years of data as reference period
+        max_year = maximum(party_rows.year)
+        ref_rows = filter(r -> r.year >= max_year - 20, party_rows)
+        if nrow(ref_rows) == 0
+            ref_rows = party_rows
+        end
+
+        n_checked += 1
+
+        mean_econ = mean(ref_rows.economic_lr)
+        mean_galtan = mean(ref_rows.galtan)
+
+        econ_ok = anchor.econ[1] <= mean_econ <= anchor.econ[2]
+        galtan_ok = anchor.galtan[1] <= mean_galtan <= anchor.galtan[2]
+        all_ok = econ_ok && galtan_ok
+
+        if all_ok
+            n_passed += 1
+        end
+
+        status = all_ok ? "PASS" : "WARN"
+        econ_marker = econ_ok ? "" : "*"
+        galtan_marker = galtan_ok ? "" : "*"
+
+        @printf("   %-15s econ=%.2f%s [%.2f-%.2f]  galtan=%.2f%s [%.2f-%.2f]  %s\n",
+            anchor.name, mean_econ, econ_marker, anchor.econ[1], anchor.econ[2],
+            mean_galtan, galtan_marker, anchor.galtan[1], anchor.galtan[2], status)
+    end
+
+    if n_checked > 0
+        println("   Anchor check: $n_passed/$n_checked within expected ranges")
+        println("   Note: Model integrates text + expert data; deviations from expert-only expectations are normal")
+    end
+
+    # Check 3: Coverage check
+    println("\n3. Coverage check:")
+    println("   Total segment-year positions: $(nrow(output))")
+    println("   Unique parties: $(length(unique(output.party_id)))")
+    blank_country_rows = count(ismissing, output.country)
+    println("   Unique countries: $(length(unique(skipmissing(output.country))))")
+    if blank_country_rows == 0
+        println("   Blank country rows: 0 - PASS")
+    else
+        println("   Blank country rows: $blank_country_rows - FAIL")
+        all_passed = false
+    end
+    println("   Year range: $(minimum(output.year)) - $(maximum(output.year))")
+
+    # V10: Check segment distribution
+    segment_counts = combine(groupby(output, :party_id), nrow => :n_years)
+    parties_multi_segment = filter(r -> r.party_id in
+        [p for p in unique(output.party_id) if length(unique(filter(x -> x.party_id == p, output).segment_num)) > 1],
+        segment_counts)
+    if nrow(parties_multi_segment) > 0
+        println("\n   Parties with multiple segments: $(length(unique(parties_multi_segment.party_id)))")
+    end
+
+    # Check 4: No duplicates (party_id, segment_num, year should be unique)
+    println("\n4. Duplicate check:")
+    dup_count = nrow(output) - nrow(unique(select(output, :party_id, :segment_num, :year)))
+    if dup_count == 0
+        println("   No duplicate (party_id, segment_num, year) combinations - PASS")
+    else
+        println("   WARNING: Found $dup_count duplicate combinations!")
+        all_passed = false
+    end
+
+    # Check 5: SE reasonableness
+    println("\n5. Standard error check:")
+    se_cols = has_4d ?
+        [:pro_market_se, :pro_welfare_se, :cosmopolitan_se, :traditional_se] :
+        [:economic_lr_se, :galtan_se]
+
+    for col in se_cols
+        if !hasproperty(output, col)
+            continue
+        end
+        vals = output[!, col]
+        mean_se = mean(vals)
+        max_se = maximum(vals)
+        println("   $col: mean=$(@sprintf("%.4f", mean_se)), max=$(@sprintf("%.4f", max_se))")
+    end
+
+    println("\n" * "-"^60)
+    if all_passed
+        println("All validation checks PASSED")
+    else
+        println("Some validation checks FAILED - please review output carefully")
+    end
+
+    return all_passed
+end
+
+# =============================================================================
+# STEP 6: Save output
+# =============================================================================
+
+function save_output(output::DataFrame, metadata::Dict, segment_info::Union{DataFrame, Nothing}, run_dir::String; outdir::String="outputs/estimations/latest")
+    println("\n" * "="^60)
+    println("SAVING OUTPUT")
+    println("="^60)
+
+    timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")
+    mkpath(outdir)
+
+    # Delete previous output files
+    for f in readdir(outdir)
+        if startswith(f, "party_positions_") && (endswith(f, ".csv") || endswith(f, ".txt") || endswith(f, ".tex"))
+            rm(joinpath(outdir, f))
+            println("  Deleted old: $f")
+        end
+    end
+
+    # Save main CSV
+    csv_file = joinpath(outdir, "party_positions_$timestamp.csv")
+    CSV.write(csv_file, output)
+    println("Saved: $csv_file")
+    println("  Rows: $(nrow(output))")
+    println("  Columns: $(ncol(output))")
+
+    # Count parties with multiple segments
+    n_multi_segment = 0
+    if segment_info !== nothing
+        party_segment_counts = combine(groupby(segment_info, :party_id), nrow => :n_segments)
+        n_multi_segment = count(r -> r.n_segments > 1, eachrow(party_segment_counts))
+    end
+
+    # Save metadata
+    meta_file = joinpath(outdir, "party_positions_$(timestamp)_metadata.txt")
+    open(meta_file, "w") do f
+        println(f, "Party Positions Dataset - Metadata")
+        println(f, "="^50)
+        println(f, "")
+        println(f, "Generated: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))")
+        println(f, "Source run: $(basename(run_dir))")
+        println(f, "Model file: $(get(metadata, "model_file", "unknown"))")
+        println(f, "")
+        println(f, "Dataset size:")
+        println(f, "  Segment-year observations: $(nrow(output))")
+        println(f, "  Unique parties: $(length(unique(output.party_id)))")
+        if n_multi_segment > 0
+            println(f, "  Parties with multiple segments: $n_multi_segment")
+        end
+        println(f, "  Unique countries: $(length(unique(output.country)))")
+        println(f, "  Year range: $(minimum(output.year)) - $(maximum(output.year))")
+        println(f, "")
+        println(f, "Columns:")
+        println(f, "  party_id: PartyFacts ID (integer) - individual party (e.g., CDU=1375, CSU=1731)")
+        println(f, "  union_party_id: PartyFacts ID of parent union (NA for standalone parties)")
+        println(f, "  segment_num: Segment number within party (1, 2, 3...)")
+        println(f, "  country: ISO2 country code")
+        println(f, "  year: Calendar year")
+        println(f, "")
+        println(f, "Segment-Based Indexing:")
+        println(f, "  - Parties are split into segments at gaps > 7 years")
+        println(f, "  - Each segment is estimated independently (no continuity across gaps)")
+        println(f, "  - Segments with < 3 observations are dropped")
+        println(f, "  - segment_num=1 is the main segment; higher numbers indicate gaps in data")
+        println(f, "")
+
+        # Check if this is 2D or 4D output
+        is_2d = !hasproperty(output, :pro_market)
+
+        if is_2d
+            println(f, "Model: 2D Direct Bipolar")
+            println(f, "")
+            println(f, "Bipolar scales (0 = left/GAL, 1 = right/TAN):")
+            println(f, "  economic_lr: Economic left-right position (directly estimated)")
+            println(f, "  galtan: GAL-TAN cultural position (directly estimated)")
+            # Note: general_lr is computed internally for cross-dimensional anchoring
+            # but not reported as output (the two dimension-specific estimates are preferred)
+        else
+            println(f, "Model: 4D Unipolar")
+            println(f, "")
+            println(f, "Dimensions (0 = low, 1 = high):")
+            println(f, "  pro_market: Pro-market economic position")
+            println(f, "  pro_welfare: Pro-welfare state position")
+            println(f, "  cosmopolitan: Cosmopolitan/GAL cultural position")
+            println(f, "  traditional: Traditional/TAN cultural position")
+            println(f, "")
+            println(f, "Derived bipolar scales (0 = left/GAL, 1 = right/TAN):")
+            println(f, "  economic_lr: Economic left-right (derived from pro_market - pro_welfare)")
+            println(f, "  galtan: GAL-TAN (derived from traditional - cosmopolitan)")
+        end
+        println(f, "")
+        println(f, "Uncertainty columns:")
+        println(f, "  *_se: Standard error (posterior SD)")
+        println(f, "  *_q025: 2.5th percentile (lower 95% CI)")
+        println(f, "  *_q975: 97.5th percentile (upper 95% CI)")
+        println(f, "")
+        println(f, "Model convergence:")
+        println(f, "  Mean R-hat: $(get(metadata, "mean_rhat", "N/A"))")
+        println(f, "  Max R-hat: $(get(metadata, "max_rhat", "N/A"))")
+        println(f, "  Mean ESS: $(get(metadata, "mean_ess", "N/A"))")
+        println(f, "  Min ESS: $(get(metadata, "min_ess", "N/A"))")
+    end
+    println("Saved: $meta_file")
+
+
+# =============================================================================
+# STEP 5b: Verify no union/alliance IDs in output
+# =============================================================================
+
+function verify_no_unions_in_output(output::DataFrame)
+    println("\n" * "="^60)
+    println("UNION ID VERIFICATION")
+    println("="^60)
+
+    union_mapping_file = joinpath("data", "union_mapping.csv")
+    if !isfile(union_mapping_file)
+        println("  No union_mapping.csv found — skipping verification")
+        return
+    end
+
+    union_df = CSV.read(union_mapping_file, DataFrame)
+    union_pf_ids = Set(union_df.manifesto_pf_id)
+    output_pf_ids = Set(output.party_id)
+
+    violations = intersect(union_pf_ids, output_pf_ids)
+
+    if isempty(violations)
+        println("  PASS: No union/alliance PF IDs found in output")
+        println("  Checked $(length(union_pf_ids)) union IDs against $(length(output_pf_ids)) output parties")
+    else
+        println("  WARNING: $(length(violations)) union PF IDs found in output")
+        println("  (This is expected if union_mapping.csv was updated after the model run)")
+        for v in sort(collect(violations))
+            n_rows = count(r -> r.party_id == v, eachrow(output))
+            println("    PF $v: $n_rows rows")
+        end
+    end
+end
+
+# =============================================================================
+# MAIN
+# =============================================================================
+
+function main()
+    println("="^60)
+    println("POST-ESTIMATION: 4D Latent Trait Model (V10)")
+    println("="^60)
+    println("Started: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))")
+
+    # Step 0: Find run directory (CLI --run-dir or auto-detect latest)
+    run_dir = nothing
+    output_dir = nothing
+    for (i, arg) in enumerate(ARGS)
+        if arg == "--run-dir" && i < length(ARGS)
+            run_dir = ARGS[i + 1]
+        elseif startswith(arg, "--run-dir=")
+            run_dir = split(arg, "=", limit=2)[2]
+        elseif arg == "--output-dir" && i < length(ARGS)
+            output_dir = ARGS[i + 1]
+        elseif startswith(arg, "--output-dir=")
+            output_dir = split(arg, "=", limit=2)[2]
+        end
+    end
+    if run_dir === nothing
+        run_dir = find_latest_run()
+    else
+        println("Using specified run directory: $run_dir")
+    end
+
+    # Step 1: Load run data
+    data = load_run_data(run_dir)
+
+    # Step 2: Build segment-year mapping
+    year0 = data.metadata["year0"]
+    segment_year_map, R, segment_info = build_segment_year_map(
+        data.text_data, data.expert_dim, data.expert_lr,
+        data.segment_info, data.segment_year_map, data.run_dir, year0
+    )
+
+    # Step 3: Load chains
+    chains = load_chains(data.chain_files)
+
+    # Step 4: Extract estimates
+    output = extract_estimates(chains, segment_year_map, R)
+
+    # Step 5: Validate
+    validate_output(output, segment_info)
+
+    # Step 5b: Verify no union/alliance IDs in output
+    verify_no_unions_in_output(output)
+
+    # Step 6: Save output
+    effective_output_dir = output_dir !== nothing ? output_dir : "outputs/estimations/latest"
+    csv_file, meta_file = save_output(output, data.metadata, segment_info, run_dir; outdir=effective_output_dir)
+
+    println("\n" * "="^60)
+    println("COMPLETE")
+    println("="^60)
+    println("Output files:")
+    println("  $csv_file")
+    println("  $meta_file")
+    println("\nFinished: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))")
+
+    return output
+end
+
+# Run if executed directly
+if abspath(PROGRAM_FILE) == @__FILE__
+    main()
+end

src/julia/pipeline/00_validation.jl

@@ -0,0 +1,332 @@
+#!/usr/bin/env julia
+#############################################################################
+## 00_validation.jl
+## Pre-flight validation checks for Stan data and initialization
+## Prevents cryptic Stan errors by catching issues early
+#############################################################################
+
+using Statistics
+
+"""
+Validate Stan data dictionary before passing to Stan.
+Catches common issues that cause Stan to crash with cryptic errors.
+"""
+function validate_stan_data(dat::Dict; verbose=true)
+    verbose && println("\n" * "="^70)
+    verbose && println("VALIDATING STAN DATA")
+    verbose && println("="^70)
+
+    errors = String[]
+    warnings = String[]
+
+    # Check for NaN/Inf in all numeric data
+    for (key, value) in dat
+        if isa(value, AbstractArray) && eltype(value) <: Number
+            if any(isnan, value)
+                push!(errors, "Data '$key' contains NaN values")
+            end
+            if any(isinf, value)
+                push!(errors, "Data '$key' contains Inf values")
+            end
+        elseif isa(value, Number)
+            if isnan(value)
+                push!(errors, "Data '$key' is NaN")
+            end
+            if isinf(value)
+                push!(errors, "Data '$key' is Inf")
+            end
+        end
+    end
+
+    # Validate expert data is in open interval (0, 1)
+    if haskey(dat, "val_dim")
+        val_dim = dat["val_dim"]
+        if any(x -> x <= 0 || x >= 1, val_dim)
+            n_boundary = count(x -> x <= 0 || x >= 1, val_dim)
+            push!(errors, "Expert dimension data has $n_boundary values at boundaries (must be in (0,1))")
+            if verbose
+                println("  Dimension-specific expert data range: [$(minimum(val_dim)), $(maximum(val_dim))]")
+            end
+        end
+    end
+
+    if haskey(dat, "val_lr")
+        val_lr = dat["val_lr"]
+        if any(x -> x <= 0 || x >= 1, val_lr)
+            n_boundary = count(x -> x <= 0 || x >= 1, val_lr)
+            push!(errors, "Expert L-R data has $n_boundary values at boundaries (must be in (0,1))")
+            if verbose
+                println("  L-R expert data range: [$(minimum(val_lr)), $(maximum(val_lr))]")
+            end
+        end
+    end
+
+    # Validate manifesto data
+    if haskey(dat, "positive") && haskey(dat, "sample")
+        positive = dat["positive"]
+        sample = dat["sample"]
+
+        if any(positive .> sample)
+            push!(errors, "Manifesto: positive counts exceed sample sizes")
+        end
+
+        if any(positive .< 0)
+            push!(errors, "Manifesto: negative positive counts found")
+        end
+
+        if any(sample .< 0)
+            push!(errors, "Manifesto: negative sample sizes found")
+        end
+    end
+
+    # Validate indices are within bounds
+    # V10: Check segment indices (ss_man) if present, otherwise party indices (jj_man)
+    if haskey(dat, "S") && haskey(dat, "ss_man")
+        S = dat["S"]
+        ss_man = dat["ss_man"]
+        if any(ss_man .< 1) || any(ss_man .> S)
+            push!(errors, "Manifesto segment indices out of bounds [1, $S]")
+        end
+    elseif haskey(dat, "J") && haskey(dat, "jj_man")
+        J = dat["J"]
+        jj_man = dat["jj_man"]
+        if any(jj_man .< 1) || any(jj_man .> J)
+            push!(errors, "Manifesto party indices out of bounds [1, $J]")
+        end
+    end
+
+    if haskey(dat, "R") && haskey(dat, "rr_man")
+        R = dat["R"]
+        rr_man = dat["rr_man"]
+        if any(rr_man .< 1) || any(rr_man .> R)
+            push!(errors, "Manifesto party-year indices out of bounds [1, $R]")
+        end
+    end
+
+    # V4: Validate constituent arrays
+    if haskey(dat, "const_rr_man") && haskey(dat, "R")
+        R = dat["R"]
+        const_rr = dat["const_rr_man"]
+        if any(const_rr .< 1) || any(const_rr .> R)
+            push!(errors, "const_rr_man out of bounds [1, $R]")
+        end
+        # Verify offsets are valid
+        if haskey(dat, "const_offset_man") && haskey(dat, "n_const_man")
+            offsets = dat["const_offset_man"]
+            n_consts = dat["n_const_man"]
+            total = dat["N_const_man_total"]
+            for i in eachindex(offsets)
+                if offsets[i] + n_consts[i] - 1 > total
+                    push!(errors, "const_offset_man[$i] + n_const_man[$i] exceeds N_const_man_total")
+                    break
+                end
+            end
+        end
+    end
+
+    # Print summary
+    if verbose
+        println("\nDATA SUMMARY:")
+        # V10: Show segments if present
+        if haskey(dat, "S")
+            println("  Segments (S): $(dat["S"])")
+            println("  Parties with segments (J): $(get(dat, "J", "N/A"))")
+        else
+            println("  Parties (J): $(get(dat, "J", "N/A"))")
+        end
+        println("  Countries (P): $(get(dat, "P", "N/A"))")
+        println("  Segment-years (R): $(get(dat, "R", "N/A"))")
+        println("  Years (T_year): $(get(dat, "T_year", "N/A"))")
+        println("  Manifesto obs: $(get(dat, "N_man", "N/A"))")
+        println("  Expert dim obs: $(get(dat, "N_exp_dim", "N/A"))")
+        println("  Expert L-R obs: $(get(dat, "N_exp_lr", "N/A"))")
+
+        if haskey(dat, "mn_resp_log_man")
+            println("\nPRIOR MEANS:")
+            println("  Manifesto: $(round(dat["mn_resp_log_man"], digits=3))")
+            println("  Expert dim: $(round(dat["mn_resp_log_exp_dim"], digits=3))")
+            println("  Expert L-R: $(round(dat["mn_resp_log_exp_lr"], digits=3))")
+        end
+    end
+
+    # Report results
+    if !isempty(errors)
+        println("\n❌ VALIDATION FAILED - $(length(errors)) ERROR(S):")
+        for (i, err) in enumerate(errors)
+            println("  $i. $err")
+        end
+        return false
+    end
+
+    if !isempty(warnings)
+        println("\n⚠️  $(length(warnings)) WARNING(S):")
+        for (i, warn) in enumerate(warnings)
+            println("  $i. $warn")
+        end
+    end
+
+    if verbose
+        println("\n✓ DATA VALIDATION PASSED")
+        println("="^70)
+    end
+
+    return true
+end
+
+"""
+Validate initialization values before passing to Stan.
+Checks for common issues that cause immediate Stan crashes.
+"""
+function validate_init_values(init_dict::Dict; verbose=true)
+    verbose && println("\n" * "="^70)
+    verbose && println("VALIDATING INITIALIZATION VALUES")
+    verbose && println("="^70)
+
+    errors = String[]
+    warnings = String[]
+
+    for (key, value) in init_dict
+        # Check for NaN/Inf
+        if isa(value, AbstractArray) && eltype(value) <: Number
+            if any(isnan, value)
+                push!(errors, "Init '$key' contains NaN")
+            end
+            if any(isinf, value)
+                push!(errors, "Init '$key' contains Inf")
+            end
+
+            if verbose && length(value) > 0
+                val_array = vec(value)
+                println("  $key: range [$(round(minimum(val_array), digits=3)), $(round(maximum(val_array), digits=3))]")
+            end
+        elseif isa(value, Number)
+            if isnan(value)
+                push!(errors, "Init '$key' is NaN")
+            end
+            if isinf(value)
+                push!(errors, "Init '$key' is Inf")
+            end
+
+            if verbose
+                println("  $key: $(round(value, digits=3))")
+            end
+        end
+
+        # Check positive constraints (common Stan constraints)
+        # Exception: *_raw parameters are non-centered and can be any real
+        if (contains(string(key), "sigma") || contains(string(key), "tau") || contains(string(key), "phi")) &&
+           !endswith(string(key), "_raw")
+            if isa(value, Number) && value <= 0
+                push!(errors, "Init '$key' = $value violates constraint > 0")
+            elseif isa(value, AbstractArray) && any(value .<= 0)
+                push!(errors, "Init '$key' has values ≤ 0 (violates constraint > 0)")
+            end
+        end
+
+        # Check Cholesky factors are valid
+        if contains(string(key), "L_Omega")
+            if isa(value, AbstractMatrix)
+                # Check it's lower triangular with positive diagonal
+                n = size(value, 1)
+                if size(value, 2) != n
+                    push!(errors, "Init '$key' is not square")
+                end
+                for i in 1:n
+                    if value[i, i] <= 0
+                        push!(errors, "Init '$key' has non-positive diagonal at position $i")
+                    end
+                    for j in (i+1):n
+                        if abs(value[i, j]) > 1e-10
+                            push!(warnings, "Init '$key' is not lower triangular")
+                            break
+                        end
+                    end
+                end
+            end
+        end
+
+        # Check slope parameters for positive constraint (V2/V3 feature)
+        if key == "Gamma_man_slope_raw"
+            if isa(value, AbstractArray) && any(value .< 0)
+                push!(errors, "Init 'Gamma_man_slope_raw' has negative values (must be ≥ 0)")
+            end
+        end
+    end
+
+    # Report results
+    if !isempty(errors)
+        println("\n❌ INIT VALIDATION FAILED - $(length(errors)) ERROR(S):")
+        for (i, err) in enumerate(errors)
+            println("  $i. $err")
+        end
+        return false
+    end
+
+    if !isempty(warnings)
+        println("\n⚠️  $(length(warnings)) WARNING(S):")
+        for (i, warn) in enumerate(warnings)
+            println("  $i. $warn")
+        end
+    end
+
+    if verbose
+        println("\n✓ INIT VALIDATION PASSED")
+        println("="^70)
+    end
+
+    return true
+end
+
+"""
+Estimate memory requirements for model
+"""
+function estimate_memory_requirements(dat::Dict; verbose=true, num_chains::Int=4, num_samples::Int=1000, num_threads_per_chain::Int=1)
+    if !verbose
+        return
+    end
+
+    println("\n" * "="^70)
+    println("MEMORY ESTIMATE")
+    println("="^70)
+
+    R = get(dat, "R", 0)
+    J = get(dat, "J", 0)
+    K_man = get(dat, "K_man", 0)
+    K_exp_dim = get(dat, "K_exp_dim", 0)
+    K_exp_lr = get(dat, "K_exp_lr", 0)
+    N_man = get(dat, "N_man", 0)
+    N_ciy = get(dat, "N_ciy", 0)
+    T_year = get(dat, "T_year", 0)
+
+    # Rough parameter count
+    theta_params = 4 * R
+    item_params = 4 * K_man * 2 + K_exp_dim * 3 + K_exp_lr * 2
+    strategic_params = get(dat, "P", 0) * get(dat, "K_man", 0)  # Country-item intercepts
+    other_params = 4 * J + T_year + J + N_ciy + 50
+
+    total_params = theta_params + item_params + strategic_params + other_params
+
+    # Memory estimate (very rough)
+    # Each parameter: ~8 bytes (float64) × samples × chains
+    total_draws_per_param = num_samples * num_chains
+    bytes_per_param = 8 * total_draws_per_param
+    total_mb = (total_params * bytes_per_param) / (1024 * 1024)
+
+    println("  Configuration:")
+    println("    Chains: $num_chains")
+    println("    Samples per chain: $num_samples")
+    println("    Threads per chain: $num_threads_per_chain")
+    println("    Total parallel workers: $(num_chains * num_threads_per_chain)")
+
+    println("  Total parameters: ~$(total_params)")
+    println("  Estimated memory (samples only): ~$(round(total_mb, digits=0)) MB")
+    thread_scaling = max(1, num_threads_per_chain)
+    println("  With thread overhead (×$(thread_scaling)): ~$(round(total_mb * thread_scaling, digits=0)) MB")
+    println("  With safety margin (×3): ~$(round(3 * total_mb * thread_scaling, digits=0)) MB")
+
+    if total_mb * thread_scaling * 3 > 8000
+        println("\n⚠️  WARNING: Model may require > 8GB RAM")
+    end
+
+    println("="^70)
+end

src/julia/pipeline/02_data_loading.jl

@@ -0,0 +1,277 @@
+#!/usr/bin/env julia
+#############################################################################
+## 02_data_loading.jl
+## Load and preprocess data for latent trait model
+## Loads three datasets: text_data (manifesto + PolDem), expert dimension-specific, expert general L-R
+##
+## Supports both:
+##   - 4D model (V10): type_high/type_low columns for bipolar bridges
+##   - 2D model (V1): dim_idx + direction columns for direct estimation
+#############################################################################
+
+using DataFrames, CSV, CategoricalArrays, Statistics
+
+#############################################################################
+## UNION MAPPING: Individual party estimates via mean-constituent model
+## Loads data/union_mapping.csv and builds lookup structures
+#############################################################################
+
+"""
+    load_union_mapping(project_root::String)
+
+Load union_mapping.csv and build lookup dictionaries.
+Returns (union_to_constituents, constituent_to_union) dicts.
+If file is missing or empty, returns empty dicts (backwards compatible).
+"""
+function load_union_mapping(project_root::String=".")
+    mapping_file = joinpath(project_root, "data", "union_mapping.csv")
+
+    union_to_constituents = Dict{Int, Vector{Int}}()
+    constituent_to_union = Dict{Int, Int}()
+
+    if !isfile(mapping_file)
+        println("  No union_mapping.csv found - running without union decomposition")
+        return union_to_constituents, constituent_to_union
+    end
+
+    df = CSV.read(mapping_file, DataFrame)
+    if nrow(df) == 0
+        println("  union_mapping.csv is empty - running without union decomposition")
+        return union_to_constituents, constituent_to_union
+    end
+
+    for row in eachrow(df)
+        union_id = row.manifesto_pf_id
+        expert_id = row.expert_pf_id
+
+        if !haskey(union_to_constituents, union_id)
+            union_to_constituents[union_id] = Int[]
+        end
+        if !(expert_id in union_to_constituents[union_id])
+            push!(union_to_constituents[union_id], expert_id)
+        end
+        constituent_to_union[expert_id] = union_id
+    end
+
+    println("  Union mapping loaded: $(length(union_to_constituents)) unions, $(length(constituent_to_union)) constituents")
+    return union_to_constituents, constituent_to_union
+end
+
+#############################################################################
+## SEGMENT-BASED INDEXING CONFIGURATION
+## Split parties at gaps > MAX_GAP years to avoid flat posteriors
+#############################################################################
+const MAX_GAP = 7      # Maximum years between observations within a segment
+const MIN_OBS = 2      # Minimum observations per segment (drop segments with fewer)
+
+#############################################################################
+## 2D MODEL MAPPING CONFIGURATION
+## Maps type_high/type_low pairs to dim_idx + direction
+#############################################################################
+const TYPE_TO_DIM_DIRECTION = Dict(
+    # Economic dimension: pro_market = right (+1), pro_welfare = left (-1)
+    ("pro_market", "pro_welfare") => (dim_idx=1, direction=1),   # Right
+    ("pro_welfare", "pro_market") => (dim_idx=1, direction=-1),  # Left
+
+    # Cultural dimension: traditional = TAN (+1), cosmopolitan = GAL (-1)
+    ("traditional", "cosmopolitan") => (dim_idx=2, direction=1), # TAN
+    ("cosmopolitan", "traditional") => (dim_idx=2, direction=-1) # GAL
+)
+
+# Expert dimension mapping (lrecon -> economic, galtan/cultural -> galtan)
+const EXPERT_VAR_TO_DIM = Dict(
+    "lrecon_ches" => 1,
+    "lrecon_vparty" => 1,
+    "welf_vparty" => 1,
+    "lrecon_gps" => 1,
+    "lrecon_poppa" => 1,
+    "galtan_ches" => 2,
+    "libcon_gps" => 2,
+    "immig_vparty" => 2,
+    "lgbt_vparty" => 2,
+    "culsup_vparty" => 2,
+    "relig_vparty" => 2,
+    "gender_vparty" => 2
+)
+
+function load_and_preprocess_4dim_data(start_year=1950; data_dir::String=".")
+    println("Loading 4D latent trait data files...")
+    println("Start year filter: $start_year")
+    data_dir != "." && println("Data directory: $data_dir")
+
+    # Load union mapping (check data_dir first, fall back to project root)
+    println("\nLoading union mapping...")
+    union_mapping_dir = isfile(joinpath(data_dir, "data", "union_mapping.csv")) ? data_dir : "."
+    union_to_constituents, constituent_to_union = load_union_mapping(union_mapping_dir)
+
+    # Load the three datasets
+    text_data_raw = CSV.read(joinpath(data_dir, "text_data.csv"), DataFrame)
+    expert_raw = CSV.read(joinpath(data_dir, "expert.csv"), DataFrame)
+    lr_data_raw = CSV.read(joinpath(data_dir, "lr_data.csv"), DataFrame)
+
+    # Filter to start year BEFORE calculating year0
+    text_data_raw = text_data_raw[text_data_raw.year .>= start_year, :]
+    expert_raw = expert_raw[expert_raw.year .>= start_year, :]
+    lr_data_raw = lr_data_raw[lr_data_raw.year .>= start_year, :]
+
+    println("Data filtered to $start_year onwards:")
+    println("  Text data (manifesto + PolDem): $(nrow(text_data_raw)) observations")
+    println("  Expert: $(nrow(expert_raw)) observations")
+    println("  L-R data: $(nrow(lr_data_raw)) observations")
+
+    # Define base year for relative time indexing
+    year0 = Int(minimum([minimum(text_data_raw.year), minimum(expert_raw.year), minimum(lr_data_raw.year)])) - 1
+    println("Base year set to: $year0")
+
+    # Create type mapping for the four dimensions
+    type_map = Dict(
+        "pro_market" => 1,
+        "pro_welfare" => 2,
+        "cosmopolitan" => 3,
+        "traditional" => 4
+    )
+
+    println("Type mapping: pro_market=1, pro_welfare=2, cosmopolitan=3, traditional=4")
+
+    # Process text data (manifesto + PolDem media)
+    text_data = copy(text_data_raw)
+    text_data = text_data[text_data.year .> year0, :]
+
+    # Add type indices for text items (V4/V10: bipolar bridge structure)
+    if !("type_high" in names(text_data)) || !("type_low" in names(text_data))
+        error("Text data must contain 'type_high' and 'type_low' columns with values: pro_market, pro_welfare, cosmopolitan, traditional")
+    end
+
+    text_data.type_high_idx = [type_map[t] for t in text_data.type_high]
+    text_data.type_low_idx = [type_map[t] for t in text_data.type_low]
+
+    # V1 (2D model): Add dim_idx and direction columns
+    # Maps type_high/type_low to single dimension + direction
+    dim_idx_man = Int[]
+    direction_man = Int[]
+    for row in eachrow(text_data)
+        key = (row.type_high, row.type_low)
+        if haskey(TYPE_TO_DIM_DIRECTION, key)
+            mapping = TYPE_TO_DIM_DIRECTION[key]
+            push!(dim_idx_man, mapping.dim_idx)
+            push!(direction_man, mapping.direction)
+        else
+            # Unknown mapping - this should not happen with valid data
+            error("Unknown type_high/type_low pair: $(row.type_high) / $(row.type_low)")
+        end
+    end
+    text_data.dim_idx_man = dim_idx_man
+    text_data.direction_man = direction_man
+
+    # Standard processing
+    text_data.country = categorical(text_data.country)
+    text_data.party = categorical(text_data.party)
+    text_data.var = categorical(text_data.var)
+    text_data.Year = Int.(text_data.year) .- year0
+    sort!(text_data, [:country, :party, :year, :var])
+    println("Text data processed: $(nrow(text_data)) observations with bipolar bridge structure")
+
+    # Process expert dimension-specific data (bipolar items like lrecon_ches, galtan_ches)
+    expert_dim_vars = ["lrecon_ches", "galtan_ches", "lrecon_vparty", "welf_vparty",
+                       "lrecon_gps", "libcon_gps", "lrecon_poppa",
+                       "immig_vparty", "lgbt_vparty", "culsup_vparty", "relig_vparty", "gender_vparty"]
+    expert_dim = expert_raw[in.(expert_raw.var, Ref(expert_dim_vars)), :]
+    expert_dim = expert_dim[(expert_dim.year .> year0) .& (expert_dim.val .>= 0) .& (expert_dim.val .<= 1), :]
+
+    # V5: Load integer observations, scale sizes, and expert counts for beta-binomial likelihood
+    expert_dim.val_int = Int.(expert_dim.val_int)
+    expert_dim.n_scale = Int.(expert_dim.n_scale)
+    expert_dim.n_experts = Int.(expert_dim.n_experts)
+
+    # Add type mappings for dimension-specific expert data
+    if !("type_low" in names(expert_dim)) || !("type_high" in names(expert_dim))
+        error("Expert data must contain 'type_low' and 'type_high' columns")
+    end
+
+    expert_dim.type_high_idx = [type_map[t] for t in expert_dim.type_high]
+    expert_dim.type_low_idx = [type_map[t] for t in expert_dim.type_low]
+
+    # V1 (2D model): Add dim_idx for expert dimension data
+    dim_idx_exp = Int[]
+    for row in eachrow(expert_dim)
+        var_name = string(row.var)
+        if haskey(EXPERT_VAR_TO_DIM, var_name)
+            push!(dim_idx_exp, EXPERT_VAR_TO_DIM[var_name])
+        else
+            # Fallback: infer from type_high/type_low
+            key = (row.type_high, row.type_low)
+            if haskey(TYPE_TO_DIM_DIRECTION, key)
+                push!(dim_idx_exp, TYPE_TO_DIM_DIRECTION[key].dim_idx)
+            else
+                error("Unknown expert variable: $var_name with type pair $(row.type_high) / $(row.type_low)")
+            end
+        end
+    end
+    expert_dim.dim_idx_exp = dim_idx_exp
+
+    expert_dim.country = categorical(expert_dim.country)
+    expert_dim.party = categorical(expert_dim.party)
+    expert_dim.var = categorical(expert_dim.var)
+    expert_dim.Year = Int.(expert_dim.year) .- year0
+    sort!(expert_dim, [:country, :party, :year, :var])
+    println("Expert dimension-specific data processed: $(nrow(expert_dim)) observations")
+
+    # Process expert general L-R data (cross-dimensional anchoring)
+    lr_vars = ["lr_ches", "lr_poppa", "lr_morgan"]  # General left-right items
+    expert_lr = lr_data_raw[in.(lr_data_raw.var, Ref(lr_vars)), :]
+    expert_lr = expert_lr[(expert_lr.year .> year0) .& (expert_lr.val .>= 0) .& (expert_lr.val .<= 1), :]
+
+    # V5: Load integer observations, scale sizes, and expert counts for beta-binomial likelihood
+    expert_lr.val_int = Int.(expert_lr.val_int)
+    expert_lr.n_scale = Int.(expert_lr.n_scale)
+    expert_lr.n_experts = Int.(expert_lr.n_experts)
+
+    expert_lr.country = categorical(expert_lr.country)
+    expert_lr.party = categorical(expert_lr.party)
+    expert_lr.var = categorical(expert_lr.var)
+    expert_lr.Year = Int.(expert_lr.year) .- year0
+    sort!(expert_lr, [:country, :party, :year, :var])
+    println("Expert general L-R data processed: $(nrow(expert_lr)) observations")
+
+    # Validate data integrity
+    println("\nData validation:")
+
+    # Check text data dimension pair distribution (V4: bipolar bridges)
+    type_pair_counts = combine(groupby(text_data, [:type_high, :type_low]), nrow => :count)
+    for row in eachrow(type_pair_counts)
+        println("  $(row.type_high) ↔ $(row.type_low): $(row.count) text data observations")
+    end
+
+    # Check expert dimension-specific type pairs
+    type_pair_counts = combine(groupby(expert_dim, [:type_high, :type_low]), nrow => :count)
+    for row in eachrow(type_pair_counts)
+        println("  $(row.type_high) - $(row.type_low): $(row.count) expert dimension-specific observations")
+    end
+
+    # Check general L-R items
+    lr_var_counts = combine(groupby(expert_lr, :var), nrow => :count)
+    for row in eachrow(lr_var_counts)
+        println("  $(row.var): $(row.count) general L-R observations")
+    end
+
+    # Check overlapping parties across datasets
+    text_data_parties = Set(text_data.party)
+    expert_dim_parties = Set(expert_dim.party)
+    expert_lr_parties = Set(expert_lr.party)
+
+    all_parties = union(text_data_parties, expert_dim_parties, expert_lr_parties)
+    println("\nParty coverage:")
+    println("  Total unique parties: $(length(all_parties))")
+    println("  In text data: $(length(text_data_parties))")
+    println("  In expert dimension-specific: $(length(expert_dim_parties))")
+    println("  In expert general L-R: $(length(expert_lr_parties))")
+    println("  In all three datasets: $(length(intersect(text_data_parties, expert_dim_parties, expert_lr_parties)))")
+
+    return text_data, expert_dim, expert_lr, year0, union_to_constituents, constituent_to_union
+end
+
+# Execute if run directly
+if abspath(PROGRAM_FILE) == @__FILE__
+    text_data, expert_dim, expert_lr, year0, u2c, c2u = load_and_preprocess_4dim_data()
+    println("4D data loading test completed successfully")
+end

src/julia/pipeline/03_data_preparation.jl

@@ -0,0 +1,947 @@
+#!/usr/bin/env julia
+#############################################################################
+## 03_data_preparation_4dim.jl
+## V10: Segment-based indexing to fix long gap interpolation issues
+##
+## Key change: Split parties at gaps > MAX_GAP years into independent segments.
+## Each segment has its own random walk (restarts at segment start).
+## Segments with < MIN_OBS observations are dropped.
+#############################################################################
+
+using DataFrames, CSV, CategoricalArrays, Statistics, StatsFuns
+
+# Import configuration from data loading module
+include("02_data_loading.jl")
+
+"""
+    split_party_years_into_segments(years::Vector{Int}, max_gap::Int)
+
+Split a party's observation years into segments based on gaps.
+Returns a vector of vectors, where each inner vector contains consecutive years
+with max `max_gap` years between observations.
+"""
+function split_party_years_into_segments(years::Vector{Int}, max_gap::Int)
+    if isempty(years)
+        return Vector{Vector{Int}}()
+    end
+
+    sorted_years = sort(unique(years))
+    segments = [Int[sorted_years[1]]]
+
+    for y in sorted_years[2:end]
+        if y - segments[end][end] <= max_gap
+            push!(segments[end], y)
+        else
+            # Gap too large - start new segment
+            push!(segments, [y])
+        end
+    end
+
+    return segments
+end
+
+function prepare_4dim_stan_data(manifesto, expert_dim, expert_lr, year0;
+                                union_to_constituents=Dict{Int,Vector{Int}}(),
+                                constituent_to_union=Dict{Int,Int}())
+    println("Preparing data for Stan model (Segment-based indexing)...")
+    println("  MAX_GAP = $MAX_GAP years, MIN_OBS = $MIN_OBS observations")
+
+    has_unions = !isempty(union_to_constituents)
+    if has_unions
+        println("  Union mapping: $(length(union_to_constituents)) unions, $(length(constituent_to_union)) constituents")
+    else
+        println("  No union mapping - standard party indexing")
+    end
+
+    # =========================================================================
+    # STEP 1: Collect observation years per party (union-aware)
+    # For union parties: create segments for each CONSTITUENT, not the union.
+    # Union manifesto years are assigned to all constituents.
+    # =========================================================================
+
+    # Identify which party IDs in data are unions vs standalone
+    # NOTE: levels() returns raw types (Int64 for integer party IDs).
+    # We consistently use String keys for all party lookups to avoid type mismatches.
+    manifesto_party_strs = Set(string.(levels(manifesto.party)))
+    union_ids_in_data = Set{String}()
+    if has_unions
+        for uid in keys(union_to_constituents)
+            uid_str = string(uid)
+            if uid_str in manifesto_party_strs
+                push!(union_ids_in_data, uid_str)
+            end
+        end
+        println("  Union party IDs found in manifesto data: $(length(union_ids_in_data))")
+    end
+
+    # Collect all party IDs that need segments
+    # For unions: constituents get segments; union itself does NOT
+    # For standalone: party gets segment as before
+    party_obs_years = Dict{String, Set{Int}}()
+
+    # First pass: collect non-union parties from all data sources (as strings)
+    all_data_parties = Set{String}()
+    for p in levels(manifesto.party)
+        push!(all_data_parties, string(p))
+    end
+    for p in levels(expert_dim.party)
+        push!(all_data_parties, string(p))
+    end
+    for p in levels(expert_lr.party)
+        push!(all_data_parties, string(p))
+    end
+
+    # Initialize observation years for standalone parties and constituents
+    for p in all_data_parties
+        p_int = tryparse(Int, string(p))
+        if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int) && string(p) in union_ids_in_data
+            # This is a union ID in manifesto - skip it, create entries for constituents instead
+            continue
+        end
+        party_obs_years[p] = Set{Int}()
+    end
+
+    # For unions: ensure all constituents have entries
+    if has_unions
+        for (uid, constituents) in union_to_constituents
+            uid_str = string(uid)
+            if uid_str in union_ids_in_data
+                for cid in constituents
+                    cid_str = string(cid)
+                    if !haskey(party_obs_years, cid_str)
+                        party_obs_years[cid_str] = Set{Int}()
+                    end
+                end
+            end
+        end
+    end
+
+    # Add years from manifesto
+    for row in eachrow(manifesto)
+        p_str = string(row.party)
+        p_int = tryparse(Int, p_str)
+        if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int) && p_str in union_ids_in_data
+            # Union manifesto: add year to ALL constituents
+            for cid in union_to_constituents[p_int]
+                cid_str = string(cid)
+                if haskey(party_obs_years, cid_str)
+                    push!(party_obs_years[cid_str], row.Year)
+                end
+            end
+        else
+            if haskey(party_obs_years, p_str)
+                push!(party_obs_years[p_str], row.Year)
+            end
+        end
+    end
+
+    # Add years from expert_dim (individual party data - direct)
+    for row in eachrow(expert_dim)
+        p_str = string(row.party)
+        if haskey(party_obs_years, p_str)
+            push!(party_obs_years[p_str], row.Year)
+        end
+    end
+
+    # Add years from expert_lr (individual party data - direct)
+    for row in eachrow(expert_lr)
+        p_str = string(row.party)
+        if haskey(party_obs_years, p_str)
+            push!(party_obs_years[p_str], row.Year)
+        end
+    end
+
+    J_original = length(party_obs_years)
+    println("Total number of parties to create segments for: $J_original")
+
+    # =========================================================================
+    # STEP 2: Split parties into segments and filter by MIN_OBS
+    # =========================================================================
+    println("\nCreating segments (splitting at gaps > $MAX_GAP years)...")
+
+    segment_data = []
+    segment_id = 0
+    parties_split = 0
+    segments_dropped = 0
+    observations_dropped = 0
+
+    for (p, years_set) in party_obs_years
+        years = collect(years_set)
+        if isempty(years)
+            continue
+        end
+
+        segments = split_party_years_into_segments(years, MAX_GAP)
+
+        if length(segments) > 1
+            parties_split += 1
+        end
+
+        for (seg_num, seg_years) in enumerate(segments)
+            n_obs = length(seg_years)
+            if n_obs >= MIN_OBS
+                segment_id += 1
+                push!(segment_data, (
+                    segment_id = segment_id,
+                    party_id = p,
+                    segment_num = seg_num,
+                    year_start = minimum(seg_years),
+                    year_end = maximum(seg_years),
+                    n_obs = n_obs
+                ))
+            else
+                segments_dropped += 1
+                observations_dropped += n_obs
+            end
+        end
+    end
+
+    segment_info = DataFrame(segment_data)
+    S = nrow(segment_info)  # Number of valid segments
+
+    println("  Segments created: $S (from $J_original parties)")
+    println("  Parties split into multiple segments: $parties_split")
+    println("  Segments dropped (< $MIN_OBS obs): $segments_dropped")
+    println("  Observations dropped: $observations_dropped")
+
+    # Get unique parties that have at least one valid segment
+    all_parties = unique(segment_info.party_id)
+    J = length(all_parties)
+    println("  Parties with valid segments: $J")
+
+    # Create party-to-index mapping for the valid parties
+    party_to_index = Dict(all_parties .=> 1:J)
+
+    # =========================================================================
+    # STEP 3: Create segment-year index space (R) - consecutive within segment
+    # =========================================================================
+    println("\nCreating segment-year index space...")
+
+    segment_year_data = []
+    for row in eachrow(segment_info)
+        for y in row.year_start:row.year_end
+            push!(segment_year_data, (
+                segment_id = row.segment_id,
+                party_id = row.party_id,
+                Year = y
+            ))
+        end
+    end
+
+    segment_year = DataFrame(segment_year_data)
+    segment_year.rr = 1:nrow(segment_year)
+    R = nrow(segment_year)
+    println("Total segment-year positions (R): $R")
+
+    # Compute len_theta_ts for segments (years per segment)
+    len_theta_ts = [row.year_end - row.year_start + 1 for row in eachrow(segment_info)]
+    @assert sum(len_theta_ts) == R "sum(len_theta_ts)=$(sum(len_theta_ts)) must equal R=$R"
+
+    # =========================================================================
+    # STEP 4: Map observations to segment-year indices (union-aware)
+    # =========================================================================
+    println("\nMapping observations to segment-year indices...")
+
+    # Create lookup: (party_str, year) -> segment_id (for valid segments only)
+    party_year_to_segment = Dict{Tuple{String, Int}, Int}()
+    for row in eachrow(segment_info)
+        for y in row.year_start:row.year_end
+            party_year_to_segment[(string(row.party_id), y)] = row.segment_id
+        end
+    end
+
+    # --- MANIFESTO: union-aware mapping ---
+    # For union manifesto obs: map to first constituent's segment (for ss_man).
+    # The actual theta averaging is handled via constituent arrays.
+    manifesto_segment_ids = Union{Int, Missing}[]
+    for row in eachrow(manifesto)
+        p_str = string(row.party)
+        p_int = tryparse(Int, p_str)
+        key = (p_str, row.Year)
+
+        if haskey(party_year_to_segment, key)
+            # Direct mapping (non-union or constituent with own segment)
+            push!(manifesto_segment_ids, party_year_to_segment[key])
+        elseif p_int !== nothing && has_unions && haskey(union_to_constituents, p_int)
+            # Union party: use first constituent's segment
+            found = false
+            for cid in union_to_constituents[p_int]
+                ckey = (string(cid), row.Year)
+                if haskey(party_year_to_segment, ckey)
+                    push!(manifesto_segment_ids, party_year_to_segment[ckey])
+                    found = true
+                    break
+                end
+            end
+            if !found
+                push!(manifesto_segment_ids, missing)
+            end
+        else
+            push!(manifesto_segment_ids, missing)
+        end
+    end
+    manifesto.segment_id = manifesto_segment_ids
+    n_manifesto_before = nrow(manifesto)
+    manifesto = manifesto[.!ismissing.(manifesto.segment_id), :]
+    manifesto.segment_id = Int.(manifesto.segment_id)
+    println("  Manifesto: $(nrow(manifesto))/$n_manifesto_before observations (dropped $(n_manifesto_before - nrow(manifesto)) in invalid segments)")
+
+    # --- EXPERT DIM: direct mapping (individual party data) ---
+    expert_dim_segment_ids = Union{Int, Missing}[]
+    for row in eachrow(expert_dim)
+        key = (string(row.party), row.Year)
+        if haskey(party_year_to_segment, key)
+            push!(expert_dim_segment_ids, party_year_to_segment[key])
+        else
+            push!(expert_dim_segment_ids, missing)
+        end
+    end
+    expert_dim.segment_id = expert_dim_segment_ids
+    n_expert_dim_before = nrow(expert_dim)
+    expert_dim = expert_dim[.!ismissing.(expert_dim.segment_id), :]
+    expert_dim.segment_id = Int.(expert_dim.segment_id)
+    println("  Expert dim: $(nrow(expert_dim))/$n_expert_dim_before observations (dropped $(n_expert_dim_before - nrow(expert_dim)) in invalid segments)")
+
+    # --- EXPERT LR: direct mapping (individual party data) ---
+    expert_lr_segment_ids = Union{Int, Missing}[]
+    for row in eachrow(expert_lr)
+        key = (string(row.party), row.Year)
+        if haskey(party_year_to_segment, key)
+            push!(expert_lr_segment_ids, party_year_to_segment[key])
+        else
+            push!(expert_lr_segment_ids, missing)
+        end
+    end
+    expert_lr.segment_id = expert_lr_segment_ids
+    n_expert_lr_before = nrow(expert_lr)
+    expert_lr = expert_lr[.!ismissing.(expert_lr.segment_id), :]
+    expert_lr.segment_id = Int.(expert_lr.segment_id)
+    println("  Expert LR: $(nrow(expert_lr))/$n_expert_lr_before observations (dropped $(n_expert_lr_before - nrow(expert_lr)) in invalid segments)")
+
+    # =========================================================================
+    # STEP 5: Create segment indices (ss) for each observation
+    # ss indexes into 1:S (segment space), used for party-level parameters
+    # =========================================================================
+
+    # Create segment_id to ss mapping
+    segment_to_ss = Dict(row.segment_id => i for (i, row) in enumerate(eachrow(segment_info)))
+
+    manifesto.ss_man = [segment_to_ss[sid] for sid in manifesto.segment_id]
+    expert_dim.ss_exp_dim = [segment_to_ss[sid] for sid in expert_dim.segment_id]
+    expert_lr.ss_exp_lr = [segment_to_ss[sid] for sid in expert_lr.segment_id]
+
+    # Validate segment indices
+    @assert all(1 .<= manifesto.ss_man .<= S)
+    @assert all(1 .<= expert_dim.ss_exp_dim .<= S)
+    @assert all(1 .<= expert_lr.ss_exp_lr .<= S)
+
+    # =========================================================================
+    # STEP 6: Map rr indices (segment-year) to datasets via leftjoin
+    # =========================================================================
+
+    # Create (segment_id, Year) -> rr lookup
+    seg_year_to_rr = Dict{Tuple{Int, Int}, Int}()
+    for row in eachrow(segment_year)
+        seg_year_to_rr[(row.segment_id, row.Year)] = row.rr
+    end
+
+    # Join manifesto with segment_year to get rr indices
+    manifesto = leftjoin(manifesto, segment_year, on=[:segment_id, :Year])
+    rename!(manifesto, :rr => :rr_man)
+
+    expert_dim = leftjoin(expert_dim, segment_year, on=[:segment_id, :Year])
+    rename!(expert_dim, :rr => :rr_exp_dim)
+
+    expert_lr = leftjoin(expert_lr, segment_year, on=[:segment_id, :Year])
+    rename!(expert_lr, :rr => :rr_exp_lr)
+
+    # Validate rr mappings
+    @assert all(!ismissing, manifesto.rr_man) "Some manifesto observations have no rr_man mapping"
+    @assert all(!ismissing, expert_dim.rr_exp_dim) "Some expert_dim observations have no rr_exp_dim mapping"
+    @assert all(!ismissing, expert_lr.rr_exp_lr) "Some expert_lr observations have no rr_exp_lr mapping"
+
+    # Convert to Int
+    manifesto.rr_man = Int.(manifesto.rr_man)
+    expert_dim.rr_exp_dim = Int.(expert_dim.rr_exp_dim)
+    expert_lr.rr_exp_lr = Int.(expert_lr.rr_exp_lr)
+
+    # Validate bounds
+    @assert all(1 .<= manifesto.rr_man .<= R) "rr_man out of bounds"
+    @assert all(1 .<= expert_dim.rr_exp_dim .<= R) "rr_exp_dim out of bounds"
+    @assert all(1 .<= expert_lr.rr_exp_lr .<= R) "rr_exp_lr out of bounds"
+
+    # Print diagnostics
+    n_observed = length(unique(vcat(manifesto.rr_man, expert_dim.rr_exp_dim, expert_lr.rr_exp_lr)))
+    println("\nSegment-years with data: $n_observed / $R ($(round(100*n_observed/R, digits=1))%)")
+    println("Segment-years for interpolation: $(R - n_observed)")
+
+    # =========================================================================
+    # STEP 6b: Build constituent arrays for union manifesto/expert observations
+    # For each manifesto obs: store list of constituent rr indices for averaging
+    # Non-union obs: single rr (n_const=1)
+    # Union obs: multiple rr values (n_const=len(constituents))
+    # =========================================================================
+    println("\nBuilding constituent arrays for mean-constituent model...")
+
+    # --- Manifesto constituent arrays ---
+    n_const_man_vec = Int[]       # n_const for each manifesto obs
+    const_rr_man_vec = Int[]      # flat array of constituent rr values
+    const_offset_man_vec = Int[]  # offset into const_rr for each obs
+
+    offset = 1
+    for row in eachrow(manifesto)
+        p_str = string(row.party)
+        p_int = tryparse(Int, p_str)
+
+        if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int) && p_str in union_ids_in_data
+            # Union manifesto obs: find rr for each constituent in this year
+            constituent_rrs = Int[]
+            for cid in union_to_constituents[p_int]
+                ckey = (string(cid), row.Year)
+                if haskey(party_year_to_segment, ckey)
+                    sid = party_year_to_segment[ckey]
+                    rr_key = (sid, row.Year)
+                    if haskey(seg_year_to_rr, rr_key)
+                        push!(constituent_rrs, seg_year_to_rr[rr_key])
+                    end
+                end
+            end
+            if isempty(constituent_rrs)
+                # Fallback: use the rr_man already assigned
+                push!(constituent_rrs, row.rr_man)
+            end
+            push!(n_const_man_vec, length(constituent_rrs))
+            push!(const_offset_man_vec, offset)
+            append!(const_rr_man_vec, constituent_rrs)
+            offset += length(constituent_rrs)
+        else
+            # Non-union: single constituent (itself)
+            push!(n_const_man_vec, 1)
+            push!(const_offset_man_vec, offset)
+            push!(const_rr_man_vec, row.rr_man)
+            offset += 1
+        end
+    end
+
+    N_const_man_total = length(const_rr_man_vec)
+    n_union_man = count(x -> x > 1, n_const_man_vec)
+    println("  Manifesto: $(nrow(manifesto)) obs, $n_union_man union obs, $N_const_man_total total constituent entries")
+
+    # --- Expert dim constituent arrays ---
+    # Individual expert obs always have n_const=1 (direct mapping)
+    # Union-level expert obs (if any) would average — but typically expert data is at individual party level
+    n_const_exp_dim_vec = Int[]
+    const_rr_exp_dim_vec = Int[]
+    const_offset_exp_dim_vec = Int[]
+
+    offset = 1
+    for row in eachrow(expert_dim)
+        p_str = string(row.party)
+        p_int = tryparse(Int, p_str)
+
+        if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int) && p_str in union_ids_in_data
+            # Union-level expert obs: average over constituents
+            constituent_rrs = Int[]
+            for cid in union_to_constituents[p_int]
+                ckey = (string(cid), row.Year)
+                if haskey(party_year_to_segment, ckey)
+                    sid = party_year_to_segment[ckey]
+                    rr_key = (sid, row.Year)
+                    if haskey(seg_year_to_rr, rr_key)
+                        push!(constituent_rrs, seg_year_to_rr[rr_key])
+                    end
+                end
+            end
+            if isempty(constituent_rrs)
+                push!(constituent_rrs, row.rr_exp_dim)
+            end
+            push!(n_const_exp_dim_vec, length(constituent_rrs))
+            push!(const_offset_exp_dim_vec, offset)
+            append!(const_rr_exp_dim_vec, constituent_rrs)
+            offset += length(constituent_rrs)
+        else
+            # Individual party obs
+            push!(n_const_exp_dim_vec, 1)
+            push!(const_offset_exp_dim_vec, offset)
+            push!(const_rr_exp_dim_vec, row.rr_exp_dim)
+            offset += 1
+        end
+    end
+
+    N_const_exp_dim_total = length(const_rr_exp_dim_vec)
+    n_union_exp_dim = count(x -> x > 1, n_const_exp_dim_vec)
+    println("  Expert dim: $(nrow(expert_dim)) obs, $n_union_exp_dim union obs, $N_const_exp_dim_total total constituent entries")
+
+    # --- Expert LR constituent arrays ---
+    n_const_exp_lr_vec = Int[]
+    const_rr_exp_lr_vec = Int[]
+    const_offset_exp_lr_vec = Int[]
+
+    offset = 1
+    for row in eachrow(expert_lr)
+        p_str = string(row.party)
+        p_int = tryparse(Int, p_str)
+
+        if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int) && p_str in union_ids_in_data
+            constituent_rrs = Int[]
+            for cid in union_to_constituents[p_int]
+                ckey = (string(cid), row.Year)
+                if haskey(party_year_to_segment, ckey)
+                    sid = party_year_to_segment[ckey]
+                    rr_key = (sid, row.Year)
+                    if haskey(seg_year_to_rr, rr_key)
+                        push!(constituent_rrs, seg_year_to_rr[rr_key])
+                    end
+                end
+            end
+            if isempty(constituent_rrs)
+                push!(constituent_rrs, row.rr_exp_lr)
+            end
+            push!(n_const_exp_lr_vec, length(constituent_rrs))
+            push!(const_offset_exp_lr_vec, offset)
+            append!(const_rr_exp_lr_vec, constituent_rrs)
+            offset += length(constituent_rrs)
+        else
+            push!(n_const_exp_lr_vec, 1)
+            push!(const_offset_exp_lr_vec, offset)
+            push!(const_rr_exp_lr_vec, row.rr_exp_lr)
+            offset += 1
+        end
+    end
+
+    N_const_exp_lr_total = length(const_rr_exp_lr_vec)
+    n_union_exp_lr = count(x -> x > 1, n_const_exp_lr_vec)
+    println("  Expert LR: $(nrow(expert_lr)) obs, $n_union_exp_lr union obs, $N_const_exp_lr_total total constituent entries")
+
+    # =========================================================================
+    # STEP 7: Filter manifesto items with sufficient observations
+    # =========================================================================
+    var_man_counts_df = combine(groupby(manifesto, :var), :var => length => :n_obs)
+    var_man_counts_df = var_man_counts_df[var_man_counts_df.n_obs .>= 2, :]
+    var_man_counts = var_man_counts_df.var
+
+    var_exp_dim_counts_df = combine(groupby(expert_dim, :var), :var => length => :n_obs)
+    var_exp_dim_counts_df = var_exp_dim_counts_df[var_exp_dim_counts_df.n_obs .>= 2, :]
+    var_exp_dim_counts = var_exp_dim_counts_df.var
+
+    var_exp_lr_counts_df = combine(groupby(expert_lr, :var), :var => length => :n_obs)
+    var_exp_lr_counts_df = var_exp_lr_counts_df[var_exp_lr_counts_df.n_obs .>= 2, :]
+    var_exp_lr_counts = var_exp_lr_counts_df.var
+
+    manifesto = manifesto[in.(manifesto.var, Ref(var_man_counts)), :]
+    manifesto.var_man = levelcode.(categorical(manifesto.var, levels=unique(var_man_counts)))
+
+    expert_dim = expert_dim[in.(expert_dim.var, Ref(var_exp_dim_counts)), :]
+    expert_dim.var_exp_dim = levelcode.(categorical(expert_dim.var, levels=unique(var_exp_dim_counts)))
+
+    expert_lr = expert_lr[in.(expert_lr.var, Ref(var_exp_lr_counts)), :]
+    expert_lr.var_exp_lr = levelcode.(categorical(expert_lr.var, levels=unique(var_exp_lr_counts)))
+
+    # =========================================================================
+    # STEP 8: Country (group) indexing
+    # =========================================================================
+    all_groups = unique(vcat(levels(manifesto.country), levels(expert_dim.country), levels(expert_lr.country)))
+    P = length(all_groups)
+    println("Total number of unique countries (P): $P")
+
+    group_to_index = Dict(all_groups .=> 1:P)
+    manifesto.pp_man = [group_to_index[c] for c in manifesto.country]
+    expert_dim.pp_exp_dim = [group_to_index[c] for c in expert_dim.country]
+    expert_lr.pp_exp_lr = [group_to_index[c] for c in expert_lr.country]
+
+    @assert all(1 .<= manifesto.pp_man .<= P)
+    @assert all(1 .<= expert_dim.pp_exp_dim .<= P)
+    @assert all(1 .<= expert_lr.pp_exp_lr .<= P)
+
+    # =========================================================================
+    # STEP 9: Country-item-year combinations for zero-inflation model
+    # =========================================================================
+    manifesto.ciy_key = string.(manifesto.country, "_", manifesto.var_man, "_", manifesto.Year)
+    ciy_keys = unique(manifesto.ciy_key)
+    N_ciy = length(ciy_keys)
+    println("Total unique country-item-year combinations (N_ciy): $N_ciy")
+
+    ciy_key_to_index = Dict(ciy_keys .=> 1:N_ciy)
+    manifesto.ciy_idx = [ciy_key_to_index[k] for k in manifesto.ciy_key]
+    @assert all(1 .<= manifesto.ciy_idx .<= N_ciy)
+
+    # =========================================================================
+    # STEP 10: Segment-country mapping (each segment inherits from party)
+    # For constituents: inherit country from their union's manifesto data
+    # =========================================================================
+    party_country_dict = Dict{String, Int}()
+
+    # From data directly
+    for row in eachrow(manifesto)
+        p_str = string(row.party)
+        p_int = tryparse(Int, p_str)
+        c_idx = group_to_index[string(row.country)]
+        party_country_dict[p_str] = c_idx
+
+        # If union, also assign country to all constituents
+        if p_int !== nothing && has_unions && haskey(union_to_constituents, p_int)
+            for cid in union_to_constituents[p_int]
+                party_country_dict[string(cid)] = c_idx
+            end
+        end
+    end
+    for row in eachrow(expert_dim)
+        party_country_dict[string(row.party)] = group_to_index[string(row.country)]
+    end
+    for row in eachrow(expert_lr)
+        party_country_dict[string(row.party)] = group_to_index[string(row.country)]
+    end
+
+    # Map each segment to its party's country
+    segment_country_idx = Int[]
+    for row in eachrow(segment_info)
+        pid = string(row.party_id)
+        if haskey(party_country_dict, pid)
+            push!(segment_country_idx, party_country_dict[pid])
+        else
+            # Fallback: try to find via union mapping
+            pf_int = tryparse(Int, pid)
+            if pf_int !== nothing && haskey(constituent_to_union, pf_int)
+                uid = constituent_to_union[pf_int]
+                if haskey(party_country_dict, string(uid))
+                    push!(segment_country_idx, party_country_dict[string(uid)])
+                else
+                    error("Cannot find country for constituent $pid (union $uid)")
+                end
+            else
+                error("Cannot find country for party $pid")
+            end
+        end
+    end
+    @assert all(1 .<= segment_country_idx .<= P)
+    @assert length(segment_country_idx) == S
+
+    # Persist canonical country on segment mapping tables
+    segment_country = [all_groups[idx] for idx in segment_country_idx]
+    segment_info.country = segment_country
+
+    segment_country_by_id = Dict(row.segment_id => segment_country[i] for (i, row) in enumerate(eachrow(segment_info)))
+    segment_year.country = [segment_country_by_id[sid] for sid in segment_year.segment_id]
+
+    # =========================================================================
+    # STEP 11: Load party family data and map to segments
+    # =========================================================================
+    println("\nLoading party family data...")
+    party_families_file = joinpath("data", "party_families.csv")
+    if !isfile(party_families_file)
+        error("party_families.csv not found at $party_families_file")
+    end
+    party_families_df = CSV.read(party_families_file, DataFrame)
+
+    pf_to_family = Dict(row.partyfacts_id => row.family for row in eachrow(party_families_df))
+
+    all_family_names = unique(party_families_df.family)
+    family_to_idx = Dict(f => i for (i, f) in enumerate(all_family_names))
+    F = length(all_family_names)
+    println("Total number of party families (F): $F")
+    println("Family categories: ", join(all_family_names, ", "))
+
+    # Map each segment to its party's family
+    # For constituents: try own family first, fall back to union's family
+    segment_family_idx = Int[]
+    unmatched_segments = String[]
+    default_family_idx = haskey(family_to_idx, "other") ? family_to_idx["other"] : 1
+
+    for row in eachrow(segment_info)
+        pf_id = tryparse(Int, string(row.party_id))
+        if !isnothing(pf_id) && haskey(pf_to_family, pf_id)
+            family_name = pf_to_family[pf_id]
+            push!(segment_family_idx, family_to_idx[family_name])
+        elseif !isnothing(pf_id) && haskey(constituent_to_union, pf_id) && haskey(pf_to_family, constituent_to_union[pf_id])
+            # Fall back to union's family
+            family_name = pf_to_family[constituent_to_union[pf_id]]
+            push!(segment_family_idx, family_to_idx[family_name])
+        else
+            push!(segment_family_idx, default_family_idx)
+            push!(unmatched_segments, "$(row.party_id)_seg$(row.segment_num)")
+        end
+    end
+
+    if !isempty(unmatched_segments)
+        println("  Warning: $(length(unmatched_segments)) segments not matched to families (assigned to 'other')")
+        if length(unmatched_segments) <= 10
+            println("    Unmatched: ", join(unmatched_segments, ", "))
+        end
+    end
+
+    @assert all(1 .<= segment_family_idx .<= F)
+    @assert length(segment_family_idx) == S
+
+    # =========================================================================
+    # STEP 12: Find anchor segment
+    # With unions: use CDU (1375) as anchor (individual constituent)
+    # Without unions: use CDU/CSU (211) as anchor (union ID)
+    # =========================================================================
+    anchor_party_id = has_unions ? 1375 : 211
+    anchor_label = has_unions ? "CDU" : "CDU/CSU"
+    anchor_segments = filter(row -> tryparse(Int, string(row.party_id)) == anchor_party_id, segment_info)
+
+    if nrow(anchor_segments) > 0
+        # Pick segment with most observations
+        anchor_segment_idx = anchor_segments[argmax(anchor_segments.n_obs), :segment_id]
+        # Convert to ss index (1:S)
+        anchor_segment_ss = segment_to_ss[anchor_segment_idx]
+        println("  Anchor segment ($anchor_label, ID $anchor_party_id): segment $anchor_segment_ss ($(anchor_segments[argmax(anchor_segments.n_obs), :n_obs]) obs)")
+    else
+        println("  Warning: Anchor party $anchor_label (ID $anchor_party_id) not found, using segment 1")
+        anchor_segment_ss = 1
+    end
+
+    # =========================================================================
+    # STEP 13: Validation - check no long gaps remain within segments
+    # =========================================================================
+    println("\nValidating segment structure...")
+    rr_to_segment_year = Dict(row.rr => (segment_id=row.segment_id, year=row.Year) for row in eachrow(segment_year))
+    seg_obs_years = Dict(row.segment_id => Set{Int}() for row in eachrow(segment_info))
+
+    for row in eachrow(manifesto)
+        push!(seg_obs_years[row.segment_id], row.Year)
+    end
+    for row in eachrow(expert_dim)
+        push!(seg_obs_years[row.segment_id], row.Year)
+    end
+    for row in eachrow(expert_lr)
+        push!(seg_obs_years[row.segment_id], row.Year)
+    end
+
+    # Union manifesto rows contribute to every constituent through const_rr_man_vec,
+    # even though row.segment_id stores only a representative segment for indexing.
+    # Include these constituent rr values so validation matches the actual Stan data.
+    for rr in const_rr_man_vec
+        if haskey(rr_to_segment_year, rr)
+            sy = rr_to_segment_year[rr]
+            push!(seg_obs_years[sy.segment_id], sy.year)
+        end
+    end
+
+    max_internal_gap = 0
+    for (i, row) in enumerate(eachrow(segment_info))
+        seg_obs = collect(seg_obs_years[row.segment_id])
+        if length(seg_obs) > 1
+            gaps = diff(sort(unique(seg_obs)))
+            if !isempty(gaps)
+                max_gap_in_seg = maximum(gaps)
+                max_internal_gap = max(max_internal_gap, max_gap_in_seg)
+                if max_gap_in_seg > MAX_GAP
+                    @warn "Segment $i (party $(row.party_id)) has internal gap of $max_gap_in_seg years!"
+                end
+            end
+        end
+    end
+    println("  Maximum internal gap within segments: $max_internal_gap years (limit: $MAX_GAP)")
+
+    # Validate all segments have >= MIN_OBS
+    for row in eachrow(segment_info)
+        @assert row.n_obs >= MIN_OBS "Segment $(row.segment_id) has only $(row.n_obs) obs (min: $MIN_OBS)"
+    end
+    println("  All segments have >= $MIN_OBS observations: PASS")
+
+    println("\nSegment-based indexing created successfully")
+    println("  S (segments): $S")
+    println("  R (segment-years): $R")
+    println("  J (parties with valid segments): $J")
+
+    return (manifesto=manifesto, expert_dim=expert_dim, expert_lr=expert_lr,
+            segment_year=segment_year, segment_info=segment_info,
+            all_parties=all_parties, all_groups=all_groups,
+            S=S, J=J, P=P, R=R, N_ciy=N_ciy, len_theta_ts=len_theta_ts,
+            segment_country_idx=segment_country_idx, group_to_index=group_to_index,
+            F=F, segment_family_idx=segment_family_idx, anchor_segment_idx=anchor_segment_ss,
+            # Constituent arrays for mean-constituent model
+            N_const_man_total=N_const_man_total,
+            n_const_man=n_const_man_vec,
+            const_offset_man=const_offset_man_vec,
+            const_rr_man=const_rr_man_vec,
+            N_const_exp_dim_total=N_const_exp_dim_total,
+            n_const_exp_dim=n_const_exp_dim_vec,
+            const_offset_exp_dim=const_offset_exp_dim_vec,
+            const_rr_exp_dim=const_rr_exp_dim_vec,
+            N_const_exp_lr_total=N_const_exp_lr_total,
+            n_const_exp_lr=n_const_exp_lr_vec,
+            const_offset_exp_lr=const_offset_exp_lr_vec,
+            const_rr_exp_lr=const_rr_exp_lr_vec,
+            union_to_constituents=union_to_constituents,
+            constituent_to_union=constituent_to_union)
+end
+
+function finalize_4dim_stan_data(manifesto, expert_dim, expert_lr, segment_year, segment_info,
+                                all_parties, all_groups, group_to_index, year0, S, J, P, R, N_ciy,
+                                len_theta_ts, segment_country_idx, F, segment_family_idx, anchor_segment_idx;
+                                N_const_man_total=0, n_const_man=Int[], const_offset_man=Int[], const_rr_man=Int[],
+                                N_const_exp_dim_total=0, n_const_exp_dim=Int[], const_offset_exp_dim=Int[], const_rr_exp_dim=Int[],
+                                N_const_exp_lr_total=0, n_const_exp_lr=Int[], const_offset_exp_lr=Int[], const_rr_exp_lr=Int[])
+    println("Finalizing 4D Stan data structure (V10: segment-based)...")
+
+    # Map years for temporal indexing
+    years_all = sort(unique(vcat(manifesto.Year, expert_dim.Year, expert_lr.Year)))
+    T_year = length(years_all)
+    year_map = DataFrame(year_rel=years_all, year_ix=1:T_year)
+
+    # Apply year mapping to all datasets
+    manifesto = leftjoin(manifesto, year_map, on=[:Year => :year_rel])
+    rename!(manifesto, :year_ix => :year_for_man)
+
+    expert_dim = leftjoin(expert_dim, year_map, on=[:Year => :year_rel])
+    rename!(expert_dim, :year_ix => :year_for_exp_dim)
+
+    expert_lr = leftjoin(expert_lr, year_map, on=[:Year => :year_rel])
+    rename!(expert_lr, :year_ix => :year_for_exp_lr)
+
+    # Validate year assignments
+    @assert all(.!ismissing.(manifesto.year_for_man))
+    @assert all(.!ismissing.(expert_dim.year_for_exp_dim))
+    @assert all(.!ismissing.(expert_lr.year_for_exp_lr))
+    @assert all(1 .<= manifesto.year_for_man .<= T_year)
+    @assert all(1 .<= expert_dim.year_for_exp_dim .<= T_year)
+    @assert all(1 .<= expert_lr.year_for_exp_lr .<= T_year)
+
+    # Add small epsilon to prevent exact zeros and ones
+    epsilon = 1e-6
+    expert_dim.val = clamp.(expert_dim.val, epsilon, 1.0 - epsilon)
+    expert_lr.val = clamp.(expert_lr.val, epsilon, 1.0 - epsilon)
+
+    # Calculate prior means
+    man_positive_sample = manifesto.positive[manifesto.sample .> 0] ./ manifesto.sample[manifesto.sample .> 0]
+    mn_resp_log_man = StatsFuns.logit(mean(man_positive_sample))
+    mn_resp_log_exp_dim = StatsFuns.logit(mean(expert_dim.val))
+    mn_resp_log_exp_lr = StatsFuns.logit(mean(expert_lr.val))
+
+    println("Prior means calculated:")
+    println("  Manifesto: $(round(mn_resp_log_man, digits=3))")
+    println("  Expert dimension-specific: $(round(mn_resp_log_exp_dim, digits=3))")
+    println("  Expert general L-R: $(round(mn_resp_log_exp_lr, digits=3))")
+
+    # V6: Decade indexing for hierarchical L-R weights
+    expert_lr_actual_years = expert_lr.Year .+ year0
+    expert_lr_decade_raw = div.(expert_lr_actual_years, 10)
+    all_lr_decades = sort(unique(expert_lr_decade_raw))
+    lr_decade_to_index = Dict(all_lr_decades .=> 1:length(all_lr_decades))
+    dd_exp_lr = [lr_decade_to_index[d] for d in expert_lr_decade_raw]
+    D_lr = length(all_lr_decades)
+    println("  Decade indexing (V6): $D_lr decades, range $(minimum(all_lr_decades)*10)s-$(maximum(all_lr_decades)*10)s")
+
+    # Create Stan data dictionary - V10 uses S (segments) instead of J (parties)
+    dat_4dim = Dict(
+        # Common data - V10: S = number of segments
+        "S" => S,                          # NEW: Number of segments (was J)
+        "J" => J,                          # Keep J for reference (parties with valid segments)
+        "P" => P,
+        "R" => R,
+        "T_year" => T_year,
+        "len_theta_ts" => Int.(len_theta_ts),
+
+        # Segment-country mapping (V10: segments inherit country from party)
+        "segment_country" => segment_country_idx,
+
+        # Segment family data (V10: segments inherit family from party)
+        "F" => F,
+        "segment_family" => segment_family_idx,
+
+        # Anchor segment for identification (CDU/CSU segment)
+        "anchor_segment" => anchor_segment_idx,
+
+        # Manifesto data - use ss_man (segment index) instead of jj_man (party index)
+        "N_man" => nrow(manifesto),
+        "K_man" => length(unique(manifesto.var_man)),
+        "kk_man" => manifesto.var_man,
+        "ss_man" => manifesto.ss_man,      # V10: segment index (was jj_man)
+        "rr_man" => manifesto.rr_man,
+        "pp_man" => manifesto.pp_man,
+        "positive" => manifesto.positive,
+        "sample" => manifesto.sample,
+        "year_for_man" => manifesto.year_for_man,
+        "type_high_idx_man" => manifesto.type_high_idx,
+        "type_low_idx_man" => manifesto.type_low_idx,
+
+        # V1 (2D model): dimension index and direction for text data
+        "dim_idx_man" => manifesto.dim_idx_man,
+        "direction_man" => manifesto.direction_man,
+
+        # Country-item-year data for zero-inflation
+        "N_ciy" => N_ciy,
+        "ciy_idx" => manifesto.ciy_idx,
+
+        # Expert dimension-specific data
+        "N_exp_dim" => nrow(expert_dim),
+        "K_exp_dim" => length(unique(expert_dim.var_exp_dim)),
+        "kk_exp_dim" => expert_dim.var_exp_dim,
+        "ss_exp_dim" => expert_dim.ss_exp_dim,  # V10: segment index
+        "rr_exp_dim" => expert_dim.rr_exp_dim,
+        "pp_exp_dim" => expert_dim.pp_exp_dim,
+        # V5 K-scaling: use rounded sum (mean × K × n_scale) and total trials (K × n_scale)
+        "val_dim_int" => Int.(clamp.(round.(expert_dim.val .* expert_dim.n_scale .* expert_dim.n_experts), 0, expert_dim.n_scale .* expert_dim.n_experts)),
+        "n_total_exp_dim" => expert_dim.n_scale .* expert_dim.n_experts,
+        "n_experts_exp_dim" => expert_dim.n_experts,
+        "type_high_idx" => expert_dim.type_high_idx,
+        "type_low_idx" => expert_dim.type_low_idx,
+
+        # V1 (2D model): dimension index for expert dimension data
+        "dim_idx_exp" => expert_dim.dim_idx_exp,
+
+        # Expert general L-R data
+        "N_exp_lr" => nrow(expert_lr),
+        "K_exp_lr" => length(unique(expert_lr.var_exp_lr)),
+        "kk_exp_lr" => expert_lr.var_exp_lr,
+        "ss_exp_lr" => expert_lr.ss_exp_lr,  # V10: segment index
+        "rr_exp_lr" => expert_lr.rr_exp_lr,
+        "pp_exp_lr" => expert_lr.pp_exp_lr,
+        # V5 K-scaling: use rounded sum (mean × K × n_scale) and total trials (K × n_scale)
+        "val_lr_int" => Int.(clamp.(round.(expert_lr.val .* expert_lr.n_scale .* expert_lr.n_experts), 0, expert_lr.n_scale .* expert_lr.n_experts)),
+        "n_total_exp_lr" => expert_lr.n_scale .* expert_lr.n_experts,
+        "n_experts_exp_lr" => expert_lr.n_experts,
+
+        # V6: Decade indexing for hierarchical L-R weights
+        "D_lr" => D_lr,
+        "dd_exp_lr" => dd_exp_lr,
+
+        # Prior information
+        "mn_resp_log_man" => mn_resp_log_man,
+        "mn_resp_log_exp_dim" => mn_resp_log_exp_dim,
+        "mn_resp_log_exp_lr" => mn_resp_log_exp_lr,
+
+        # Constituent arrays for mean-constituent model (V4)
+        "N_const_man_total" => max(1, N_const_man_total),
+        "n_const_man" => isempty(n_const_man) ? ones(Int, nrow(manifesto)) : n_const_man,
+        "const_offset_man" => isempty(const_offset_man) ? collect(1:nrow(manifesto)) : const_offset_man,
+        "const_rr_man" => isempty(const_rr_man) ? manifesto.rr_man : const_rr_man,
+
+        "N_const_exp_dim_total" => max(1, N_const_exp_dim_total),
+        "n_const_exp_dim" => isempty(n_const_exp_dim) ? ones(Int, nrow(expert_dim)) : n_const_exp_dim,
+        "const_offset_exp_dim" => isempty(const_offset_exp_dim) ? collect(1:nrow(expert_dim)) : const_offset_exp_dim,
+        "const_rr_exp_dim" => isempty(const_rr_exp_dim) ? expert_dim.rr_exp_dim : const_rr_exp_dim,
+
+        "N_const_exp_lr_total" => max(1, N_const_exp_lr_total),
+        "n_const_exp_lr" => isempty(n_const_exp_lr) ? ones(Int, nrow(expert_lr)) : n_const_exp_lr,
+        "const_offset_exp_lr" => isempty(const_offset_exp_lr) ? collect(1:nrow(expert_lr)) : const_offset_exp_lr,
+        "const_rr_exp_lr" => isempty(const_rr_exp_lr) ? expert_lr.rr_exp_lr : const_rr_exp_lr
+    )
+
+    println("4D Stan data dictionary created with $(length(dat_4dim)) elements")
+
+    # Print summary statistics
+    println("\nData summary (V10: Segment-based):")
+    println("  Segments: $(dat_4dim["S"])")
+    println("  Parties with valid segments: $(dat_4dim["J"])")
+    println("  Segment-year combinations: $(dat_4dim["R"])")
+    println("  Manifesto observations: $(dat_4dim["N_man"])")
+    println("  Expert dimension-specific observations: $(dat_4dim["N_exp_dim"])")
+    println("  Expert general L-R observations: $(dat_4dim["N_exp_lr"])")
+    println("  Unique manifesto items: $(dat_4dim["K_man"])")
+    println("  Unique expert dimension-specific items: $(dat_4dim["K_exp_dim"])")
+    println("  Unique expert general L-R items: $(dat_4dim["K_exp_lr"])")
+    println("  Years: $(dat_4dim["T_year"])")
+
+    return (dat_4dim=dat_4dim, manifesto=manifesto, expert_dim=expert_dim,
+            expert_lr=expert_lr, T_year=T_year, segment_year=segment_year,
+            segment_info=segment_info)
+end
+
+# Execute if run directly
+if abspath(PROGRAM_FILE) == @__FILE__
+    println("Run from main script to execute the full 4D pipeline")
+end

src/julia/pipeline/05_results_processing.jl

@@ -0,0 +1,174 @@
+#!/usr/bin/env julia
+#############################################################################
+## 05_results_processing.jl
+## Extract and process 4D model results with diagnostics
+## Adapted from old_project for latent traits only (no election effects)
+#############################################################################
+
+using StanSample, DataFrames, Statistics
+
+function extract_model_results_4dim(stanmodel)
+    """
+    Extract model results for 4D latent trait model
+    Simplified version - no election effects (pure latent traits)
+    """
+    println("Extracting 4D model results...")
+
+    try
+        println("Model completed successfully - extracting results")
+
+        # For 4D latent trait model, we save the full stanmodel object
+        # Post-estimation will extract specific parameters later
+
+        return (
+            samples = stanmodel,  # Save the full stanmodel with all MCMC samples
+            extraction_status = "success"
+        )
+
+    catch e
+        println("Error in result extraction: $e")
+        return (
+            samples = "error",
+            extraction_status = "error",
+            error_message = string(e)
+        )
+    end
+end
+
+function compute_model_diagnostics(stanmodel_result)
+    """
+    Compute convergence diagnostics from Stan model
+    Returns R-hat, ESS statistics, and overall convergence assessment
+
+    stanmodel_result can be either:
+    - A SampleModel object directly
+    - A named tuple from run_4dim_stan_model containing .stanmodel
+    """
+    println("Computing model diagnostics...")
+
+    try
+        # Handle both direct SampleModel and named tuple from run_4dim_stan_model
+        stanmodel = if hasproperty(stanmodel_result, :stanmodel)
+            stanmodel_result.stanmodel
+        else
+            stanmodel_result
+        end
+
+        # Get REAL diagnostics using StanSample.jl
+        diagnostics_summary = read_summary(stanmodel)
+
+        # Extract real Rhat and ESS values. Stan summary column names differ
+        # across CmdStan/StanSample versions, so resolve aliases explicitly.
+        summary_names = names(diagnostics_summary)
+        rhat_col = if "r_hat" in summary_names
+            "r_hat"
+        elseif "R_hat" in summary_names
+            "R_hat"
+        elseif "RHat" in summary_names
+            "RHat"
+        else
+            error("No R-hat column found in summary. Columns: $(join(summary_names, ", "))")
+        end
+        ess_col = if "ess_bulk" in summary_names
+            "ess_bulk"
+        elseif "ess" in summary_names
+            "ess"
+        elseif "ESS_bulk" in summary_names
+            "ESS_bulk"
+        elseif "n_eff" in summary_names
+            "n_eff"
+        else
+            error("No ESS column found in summary. Columns: $(join(summary_names, ", "))")
+        end
+
+        rhat_vals = diagnostics_summary[!, rhat_col]
+        ess_bulk_vals = diagnostics_summary[!, ess_col]
+
+        # Compute real statistics (handle NaN values properly)
+        # Use isfinite to exclude both missing and NaN values
+        valid_rhat = filter(isfinite, rhat_vals)
+        valid_ess = filter(isfinite, ess_bulk_vals)
+
+        mean_rhat = length(valid_rhat) > 0 ? mean(valid_rhat) : NaN
+        max_rhat = length(valid_rhat) > 0 ? maximum(valid_rhat) : NaN
+        mean_ess = length(valid_ess) > 0 ? mean(valid_ess) : NaN
+        min_ess = length(valid_ess) > 0 ? minimum(valid_ess) : NaN
+
+        # Count problematic parameters (use isfinite for consistent counting)
+        high_rhat_count = count(x -> isfinite(x) && x > 1.1, rhat_vals)
+        moderate_rhat_count = count(x -> isfinite(x) && x > 1.05, rhat_vals)
+        low_ess_count = count(x -> isfinite(x) && x < 400, ess_bulk_vals)
+        very_low_ess_count = count(x -> isfinite(x) && x < 100, ess_bulk_vals)
+
+        # Total parameter count
+        total_params = length(valid_rhat)
+
+        # Overall assessment (handle NaN values)
+        if isnan(max_rhat) || total_params == 0
+            convergence_status = "insufficient_data"
+        else
+            excellent_convergence = max_rhat < 1.05 && high_rhat_count == 0 && very_low_ess_count == 0
+            good_convergence = max_rhat < 1.1 && high_rhat_count < 5 && very_low_ess_count < total_params * 0.1
+            acceptable_convergence = max_rhat < 1.2 && high_rhat_count < total_params * 0.1
+
+            if excellent_convergence
+                convergence_status = "excellent"
+            elseif good_convergence
+                convergence_status = "good"
+            elseif acceptable_convergence
+                convergence_status = "acceptable"
+            else
+                convergence_status = "poor"
+            end
+        end
+
+        println("\nDiagnostics computed:")
+        println("  Total parameters: $total_params")
+        println("  Mean R-hat: $(round(mean_rhat, digits=4))")
+        println("  Max R-hat: $(round(max_rhat, digits=4))")
+        println("  High R-hat count (>1.1): $high_rhat_count")
+        println("  Mean ESS: $(round(mean_ess, digits=0))")
+        println("  Min ESS: $(round(min_ess, digits=0))")
+        println("  Very low ESS count (<100): $very_low_ess_count")
+        println("  Convergence status: $convergence_status")
+
+        return (
+            diagnostics_summary = diagnostics_summary,
+            convergence_status = convergence_status,
+            mean_rhat = mean_rhat,
+            max_rhat = max_rhat,
+            mean_ess = mean_ess,
+            min_ess = min_ess,
+            high_rhat_count = high_rhat_count,
+            moderate_rhat_count = moderate_rhat_count,
+            low_ess_count = low_ess_count,
+            very_low_ess_count = very_low_ess_count,
+            total_params = total_params
+        )
+
+    catch e
+        println("Error in diagnostics computation: $e")
+        println("Stack trace:")
+        showerror(stdout, e, catch_backtrace())
+
+        return (
+            diagnostics_summary = "error",
+            convergence_status = "error",
+            mean_rhat = 999.0,
+            max_rhat = 999.0,
+            mean_ess = 0.0,
+            min_ess = 0.0,
+            high_rhat_count = 999,
+            moderate_rhat_count = 999,
+            low_ess_count = 999,
+            very_low_ess_count = 999,
+            total_params = 0,
+            error_message = string(e)
+        )
+    end
+end
+
+# Execute if run directly
+if abspath(PROGRAM_FILE) == @__FILE__
+    println("Run from main run_model.jl to execute the full pipeline")
+end

src/julia/pipeline/06_save_model.jl

@@ -0,0 +1,428 @@
+#!/usr/bin/env julia
+#############################################################################
+## 06_save_model.jl
+## CSV-First Save Architecture
+## Robust, portable, no serialization issues
+#############################################################################
+
+module RobustSave
+
+using Dates, Printf, CSV, DataFrames, JSON
+
+export robust_save_model_csv, robust_save_model
+
+"""
+CSV-First Save Architecture
+
+When chains_already_saved=true (BULLETPROOF MODE):
+- Chains already saved to run_dir/chains/ by model execution
+- Just verify chains and add metadata/data files
+- Even if this function crashes, chains are SAFE
+
+When chains_already_saved=false (legacy mode):
+- Copy chains from temp directory to new run directory
+- Add metadata/data files
+
+Saves model results as:
+1. CSV chain files (source of truth - never fail)
+2. Data CSVs (original inputs for reproducibility)
+3. Simple metadata.json (no complex types)
+4. Human-readable README.txt
+
+No JLD2, no serialization issues, fully portable and reproducible.
+"""
+function robust_save_model_csv(
+    run_dir_or_temp::String,
+    data_dict::Dict,
+    original_data::Dict,
+    metadata::Dict;
+    chains_already_saved::Bool=false
+)
+    println("\n" * "=" ^ 70)
+    if chains_already_saved
+        println("ADDING METADATA TO EXISTING RUN (chains already secured)")
+    else
+        println("CSV-FIRST MODEL SAVE")
+    end
+    println("=" ^ 70)
+
+    # Determine directories based on mode
+    if chains_already_saved
+        # Chains already saved - run_dir_or_temp IS the run directory
+        run_dir = run_dir_or_temp
+        chains_dir = joinpath(run_dir, "chains")
+        data_dir = joinpath(run_dir, "data")
+        run_id = basename(run_dir)
+        timestamp = replace(run_id, "run_" => "")
+
+        println("Run directory: $run_dir")
+        println("Mode: Chains already secured, adding metadata")
+
+        # Verify chains directory exists
+        if !isdir(chains_dir)
+            error("CRITICAL: Chains directory not found: $chains_dir")
+        end
+
+        # Count existing chain files
+        chain_files = filter(f -> endswith(f, ".csv") && contains(f, "chain"), readdir(chains_dir))
+        if isempty(chain_files)
+            error("CRITICAL: No chain CSV files found in $chains_dir")
+        end
+
+        println("Found $(length(chain_files)) chain files already saved")
+
+        # Calculate total size
+        total_size_gb = 0.0
+        for chain_file in chain_files
+            chain_path = joinpath(chains_dir, chain_file)
+            total_size_gb += filesize(chain_path) / (1024^3)
+        end
+
+        println("✓ Chains verified ($(round(total_size_gb, digits=2)) GB total)")
+
+    else
+        # Legacy mode - create new run directory and copy chains
+        temp_csv_dir = run_dir_or_temp
+
+        timestamp = Dates.format(Dates.now(), "yyyy-mm-dd_HH-MM-SS")
+        run_id = "run_$(timestamp)"
+        run_dir = joinpath("outputs", "model_outputs", "latest", run_id)
+        chains_dir = joinpath(run_dir, "chains")
+        data_dir = joinpath(run_dir, "data")
+
+        println("Run ID: $run_id")
+        println("Output directory: $run_dir")
+
+        # Create directory structure
+        mkpath(chains_dir)
+
+        # STEP 1: Copy CSV chain files
+        println("\n" * "=" ^ 70)
+        println("STEP 1: Copying MCMC chain CSV files")
+        println("=" ^ 70)
+        println("Source: $temp_csv_dir")
+        println("Destination: $chains_dir")
+
+        csv_files = filter(f -> endswith(f, ".csv"), readdir(temp_csv_dir))
+        chain_files = filter(f -> contains(f, "chain"), csv_files)
+
+        if isempty(chain_files)
+            error("No chain CSV files found in $temp_csv_dir")
+        end
+
+        println("Found $(length(chain_files)) chain files")
+
+        total_size_gb = 0.0
+        for (i, csv_file) in enumerate(sort(chain_files))
+            src_path = joinpath(temp_csv_dir, csv_file)
+
+            # Rename to standard format: chain_1.csv, chain_2.csv, etc.
+            dest_filename = "chain_$i.csv"
+            dest_path = joinpath(chains_dir, dest_filename)
+
+            src_size = filesize(src_path)
+            size_gb = src_size / (1024^3)
+            total_size_gb += size_gb
+
+            println("  Copying $csv_file → $dest_filename ($(round(size_gb, digits=2)) GB)")
+            cp(src_path, dest_path, force=true)
+
+            # Verify copy with size check
+            dest_size = filesize(dest_path)
+            if dest_size != src_size
+                error("CRITICAL: Size mismatch for $dest_filename! Source: $src_size, Dest: $dest_size")
+            end
+        end
+
+        println("✓ All chains copied and verified ($(round(total_size_gb, digits=2)) GB total)")
+    end
+
+    # Create data directory
+    mkpath(data_dir)
+
+    # STEP 2: Save data CSVs
+    println("\n" * "=" ^ 70)
+    println("STEP 2: Saving original data CSVs")
+    println("=" ^ 70)
+
+    for (name, df) in original_data
+        if isa(df, DataFrame)
+            csv_path = joinpath(data_dir, "$(name).csv")
+            println("  Saving $(name).csv ($(nrow(df)) rows)")
+            CSV.write(csv_path, df)
+        end
+    end
+
+    println("✓ Data CSVs saved")
+
+    # STEP 3: Save Stan data dictionary as JSON
+    println("\n" * "=" ^ 70)
+    println("STEP 3: Saving Stan data dictionary")
+    println("=" ^ 70)
+
+    # Convert data_dict to JSON-serializable format
+    stan_data_json = Dict{String, Any}()
+    for (k, v) in data_dict
+        try
+            # Only save simple types (numbers, arrays of numbers)
+            if isa(v, Number) || isa(v, AbstractArray{<:Number})
+                stan_data_json[k] = v
+            elseif isa(v, AbstractArray)
+                # Try to convert, skip if fails
+                try
+                    stan_data_json[k] = collect(v)
+                catch
+                    println("  Skipping $k (complex type)")
+                end
+            end
+        catch e
+            println("  Warning: Could not serialize $k: $e")
+        end
+    end
+
+    stan_data_path = joinpath(data_dir, "stan_data.json")
+    open(stan_data_path, "w") do f
+        JSON.print(f, stan_data_json, 2)
+    end
+    println("✓ Stan data saved to stan_data.json")
+
+    # Count chain files for metadata
+    chain_files_final = filter(f -> endswith(f, ".csv") && contains(f, "chain"), readdir(chains_dir))
+    num_chains = length(chain_files_final)
+
+    # Recalculate total_size_gb if in chains_already_saved mode
+    if chains_already_saved
+        total_size_gb = 0.0
+        for chain_file in chain_files_final
+            chain_path = joinpath(chains_dir, chain_file)
+            total_size_gb += filesize(chain_path) / (1024^3)
+        end
+    end
+
+    # STEP 4: Save metadata
+    println("\n" * "=" ^ 70)
+    println("STEP 4: Saving metadata")
+    println("=" ^ 70)
+
+    # Add run info to metadata
+    metadata["run_id"] = run_id
+    metadata["timestamp"] = timestamp
+    metadata["files"] = Dict(
+        "chains" => ["chains/chain_$i.csv" for i in 1:num_chains],
+        "data" => readdir(data_dir),
+        "chain_size_gb" => num_chains > 0 ? round(total_size_gb / num_chains, digits=2) : 0.0,
+        "total_size_gb" => round(total_size_gb, digits=2)
+    )
+
+    metadata_path = joinpath(run_dir, "metadata.json")
+    open(metadata_path, "w") do f
+        JSON.print(f, metadata, 2)
+    end
+    println("✓ Metadata saved to metadata.json")
+
+    # STEP 5: Generate README
+    println("\n" * "=" ^ 70)
+    println("STEP 5: Generating README")
+    println("=" ^ 70)
+
+    readme_path = joinpath(run_dir, "README.txt")
+    generate_readme(readme_path, run_id, metadata, num_chains, total_size_gb)
+    println("✓ README generated")
+
+    # STEP 6: Final verification
+    println("\n" * "=" ^ 70)
+    println("STEP 6: Verification")
+    println("=" ^ 70)
+
+    # Verify all chain files exist and are readable
+    all_good = true
+    verified_files = Dict{String, Dict{String, Any}}()
+
+    for i in 1:num_chains
+        chain_path = joinpath(chains_dir, "chain_$i.csv")
+        if !isfile(chain_path)
+            println("  ✗ Missing: chain_$i.csv")
+            all_good = false
+        else
+            # Quick read test and size check
+            try
+                CSV.File(chain_path; limit=1)
+                file_size_gb = filesize(chain_path) / (1024^3)
+                verified_files["chain_$i.csv"] = Dict(
+                    "path" => chain_path,
+                    "size_gb" => file_size_gb,
+                    "verified" => true
+                )
+                println("  ✓ chain_$i.csv verified ($(round(file_size_gb, digits=2)) GB)")
+            catch e
+                println("  ✗ Cannot read chain_$i.csv: $e")
+                all_good = false
+            end
+        end
+    end
+
+    if !all_good
+        error("Verification failed - some files are missing or corrupted")
+    end
+
+    println("\n" * "=" ^ 70)
+    println("✓ MODEL SAVED SUCCESSFULLY")
+    println("=" ^ 70)
+    println("Run directory: $run_dir")
+    println("Total size: $(round(total_size_gb, digits=2)) GB")
+    println("Status: All files verified and ready")
+    println("=" ^ 70)
+
+    # Return verification details for cleanup
+    return (
+        run_dir = run_dir,
+        verified_files = verified_files,
+        total_size_gb = total_size_gb,
+        verification_passed = all_good,
+        num_chains = num_chains
+    )
+end
+
+function generate_readme(
+    filepath::String,
+    run_id::String,
+    metadata::Dict,
+    num_chains::Int,
+    total_size_gb::Float64
+)
+    """Generate human-readable README file"""
+
+    open(filepath, "w") do f
+        write(f, "=" ^ 78 * "\n")
+        write(f, "4D LATENT TRAIT MODEL - MODEL RUN RESULTS\n")
+        write(f, "=" ^ 78 * "\n\n")
+
+        write(f, "Run ID: $run_id\n")
+        write(f, "Model: $(get(metadata, "model_file", "unknown"))\n")
+        write(f, "Date: $(Dates.format(Dates.now(), "yyyy-mm-dd HH:MM:SS"))\n")
+        write(f, "Status: $(get(metadata, "convergence_status", "unknown"))\n\n")
+
+        write(f, "=" ^ 78 * "\n")
+        write(f, "DIRECTORY CONTENTS\n")
+        write(f, "=" ^ 78 * "\n\n")
+
+        write(f, "chains/\n")
+        for i in 1:num_chains
+            write(f, "  ├── chain_$i.csv\n")
+        end
+        write(f, "  Total: $(get(metadata, "num_chains", num_chains)) chains × " *
+              "$(get(metadata, "num_samples", "?")) samples\n")
+        write(f, "  Size: $(round(total_size_gb, digits=2)) GB\n\n")
+
+        write(f, "data/\n")
+        write(f, "  ├── text_data.csv\n")
+        write(f, "  ├── expert_dim.csv\n")
+        write(f, "  ├── expert_lr.csv\n")
+        write(f, "  ├── segment_year_map.csv (V10)\n")
+        write(f, "  ├── segment_info.csv (V10)\n")
+        write(f, "  └── stan_data.json\n\n")
+
+        write(f, "=" ^ 78 * "\n")
+        write(f, "MODEL CONFIGURATION\n")
+        write(f, "=" ^ 78 * "\n\n")
+
+        write(f, "Chains: $(get(metadata, "num_chains", "?"))\n")
+        write(f, "Warmup: $(get(metadata, "num_warmup", "?"))\n")
+        write(f, "Samples: $(get(metadata, "num_samples", "?"))\n")
+        write(f, "Adapt delta: $(get(metadata, "adapt_delta", "?"))\n")
+        write(f, "Max depth: $(get(metadata, "max_depth", "?"))\n\n")
+
+        write(f, "Dimensions: $(join(get(metadata, "dimensions", ["?"]), ", "))\n\n")
+
+        write(f, "=" ^ 78 * "\n")
+        write(f, "HOW TO USE THESE RESULTS\n")
+        write(f, "=" ^ 78 * "\n\n")
+
+        write(f, "To extract party positions:\n\n")
+        write(f, "  julia 02_post_estimation.jl\n\n")
+
+        write(f, "This will read the CSV files and generate party_positions_[timestamp].csv\n")
+        write(f, "with uncertainty estimates (SE, credible intervals).\n\n")
+
+        write(f, "=" ^ 78 * "\n")
+        write(f, "Generated: $(Dates.format(Dates.now(), "yyyy-mm-dd HH:MM:SS"))\n")
+        write(f, "=" ^ 78 * "\n")
+    end
+end
+
+"""
+Wrapper for robust_save_model_csv that handles the stanmodel tuple from run_4dim_stan_model.
+
+The model execution now saves chains BEFORE returning, so this function:
+1. Verifies chains are already saved in final_run_dir
+2. Adds metadata and data files
+3. Returns the output path
+
+Arguments:
+- stanmodel_tuple: Named tuple from run_4dim_stan_model (contains .stanmodel, .final_run_dir, etc.)
+- model_data: Dict with data_dict, original data, and metadata
+- output_dir: Base output directory (ignored - uses stanmodel_tuple.final_run_dir)
+- compress: Ignored (CSV-first architecture)
+- keep_local_backups: Ignored (chains already saved)
+"""
+function robust_save_model(
+    stanmodel_tuple,
+    model_data::Dict,
+    output_dir::String;
+    compress::Bool=true,
+    keep_local_backups::Int=2
+)
+    # Extract the final run directory from the stanmodel tuple
+    if !hasproperty(stanmodel_tuple, :final_run_dir)
+        error("stanmodel_tuple missing :final_run_dir - chains may not be saved!")
+    end
+
+    final_run_dir = stanmodel_tuple.final_run_dir
+
+    # Prepare original data for saving
+    original_data = Dict{String, Any}()
+    if haskey(model_data, "manifesto")
+        original_data["text_data"] = model_data["manifesto"]
+    end
+    if haskey(model_data, "expert_dim")
+        original_data["expert_dim"] = model_data["expert_dim"]
+    end
+    if haskey(model_data, "expert_lr")
+        original_data["expert_lr"] = model_data["expert_lr"]
+    end
+    # V10: Save segment_year_map and segment_info for post-estimation
+    if haskey(model_data, "segment_year")
+        original_data["segment_year_map"] = model_data["segment_year"]
+    end
+    if haskey(model_data, "segment_info")
+        original_data["segment_info"] = model_data["segment_info"]
+    end
+    # V9 fallback: Save party_year_map for post-estimation (includes interpolated years)
+    if haskey(model_data, "party_year")
+        original_data["party_year_map"] = model_data["party_year"]
+    end
+
+    # Prepare metadata
+    metadata = Dict{String, Any}()
+    if haskey(model_data, "model_info")
+        for (k, v) in model_data["model_info"]
+            metadata[string(k)] = v
+        end
+    end
+
+    # Get data_dict
+    data_dict = get(model_data, "data_dict", Dict{String, Any}())
+
+    # Call the CSV-first save with chains_already_saved=true
+    result = robust_save_model_csv(
+        final_run_dir,
+        data_dict,
+        original_data,
+        metadata;
+        chains_already_saved=true
+    )
+
+    return result.run_dir
+end
+
+end  # module

src/julia/pipeline/performance_monitoring.jl

@@ -0,0 +1,150 @@
+#!/usr/bin/env julia
+#############################################################################
+## performance_monitoring.jl
+## Post-run performance diagnostics for Stan sampling jobs
+#############################################################################
+
+using StanSample
+using Statistics: mean, median
+using Dates
+
+function _safe_column(df, candidates::Vector{String})
+    for candidate in candidates
+        sym = Symbol(candidate)
+        if sym in names(df)
+            return df[!, sym]
+        elseif candidate in names(df)
+            return df[!, candidate]
+        end
+    end
+    return nothing
+end
+
+function _clean_values(vec)
+    cleaned = Float64[]
+    for v in vec
+        if v isa Missing || v === nothing
+            continue
+        end
+        try
+            value = Float64(v)
+            isfinite(value) && push!(cleaned, value)
+        catch
+            continue
+        end
+    end
+    return cleaned
+end
+
+function _summarize_vector(vec)
+    if vec === nothing
+        return Dict{String,Any}("available" => false)
+    end
+    cleaned = _clean_values(vec)
+    if isempty(cleaned)
+        return Dict{String,Any}("available" => false)
+    end
+    return Dict{String,Any}(
+        "available" => true,
+        "count" => length(cleaned),
+        "mean" => mean(cleaned),
+        "median" => median(cleaned),
+        "min" => minimum(cleaned),
+        "max" => maximum(cleaned)
+    )
+end
+
+function monitor_sampling_performance!(stanmodel;
+    run_metrics::Union{Nothing,Dict{String,Any}}=nothing,
+    metrics_path::Union{Nothing,String}=nothing,
+    csv_paths::Union{Nothing,Vector{String}}=nothing,
+    aggregate_metrics::Union{Nothing,Dict{String,Any}}=nothing,
+    max_depth::Union{Nothing,Int}=nothing)
+
+    csv_paths === nothing && (csv_paths = discover_stan_csvs([stanmodel.tmpdir]))
+    aggregate_metrics === nothing && begin
+        _, aggregate_metrics = collect_run_metrics(csv_paths; max_depth=max_depth)
+    end
+
+    summary_df = nothing
+    try
+        summary_df = read_summary(stanmodel)
+    catch e
+        println("Warning: could not read Stan summary: $e")
+    end
+
+    performance = Dict{String,Any}(
+        "generated_at" => Dates.format(Dates.now(), "yyyy-mm-ddTHH:MM:SS"),
+        "csv_paths" => csv_paths
+    )
+
+    if summary_df !== nothing
+        performance["ess_bulk"] = _summarize_vector(_safe_column(summary_df, ["ess_bulk", "ess"]))
+        performance["ess_tail"] = _summarize_vector(_safe_column(summary_df, ["ess_tail"]))
+        performance["ess_per_sec"] = _summarize_vector(_safe_column(summary_df, ["ess_per_sec", "n_eff/s"]))
+        performance["r_hat"] = _summarize_vector(_safe_column(summary_df, ["r_hat"]))
+
+        if haskey(performance["r_hat"], "available") && performance["r_hat"]["available"]
+            performance["r_hat"]["max"] = maximum(_clean_values(_safe_column(summary_df, ["r_hat"])))
+        end
+
+        performance["parameters_considered"] = size(summary_df, 1)
+    else
+        performance["ess_bulk"] = Dict{String,Any}("available" => false)
+        performance["ess_tail"] = Dict{String,Any}("available" => false)
+        performance["ess_per_sec"] = Dict{String,Any}("available" => false)
+        performance["r_hat"] = Dict{String,Any}("available" => false)
+        performance["parameters_considered"] = 0
+    end
+
+    divergences = get(aggregate_metrics, "divergences", 0)
+    total_samples = stanmodel.num_samples * stanmodel.num_chains
+    divergence_rate = total_samples > 0 ? divergences / total_samples : nothing
+
+    performance["divergences"] = Dict{String,Any}(
+        "count" => divergences,
+        "rate" => divergence_rate,
+        "total_draws" => total_samples
+    )
+
+    performance["leapfrog"] = Dict{String,Any}(
+        "mean" => get(aggregate_metrics, "mean_leapfrog", nothing)
+    )
+
+    performance["step_size"] = Dict{String,Any}(
+        "mean" => get(aggregate_metrics, "mean_step_size", nothing)
+    )
+
+    sampling_seconds = get(aggregate_metrics, "sampling_seconds", nothing)
+    if sampling_seconds !== nothing && sampling_seconds > 0
+        performance["throughput"] = Dict{String,Any}(
+            "samples_per_second" => (total_samples / sampling_seconds),
+            "seconds_sampling" => sampling_seconds
+        )
+    else
+        performance["throughput"] = Dict{String,Any}(
+            "samples_per_second" => nothing,
+            "seconds_sampling" => sampling_seconds
+        )
+    end
+
+    if run_metrics !== nothing
+        run_metrics["performance"] = performance
+        if metrics_path !== nothing
+            safe_write_json(metrics_path, run_metrics)
+        end
+    elseif metrics_path !== nothing
+        temp_metrics = Dict{String,Any}("performance" => performance)
+        safe_write_json(metrics_path, temp_metrics)
+    end
+
+    println("\nPERFORMANCE SUMMARY")
+    println("  ESS bulk (mean): $(get(performance["ess_bulk"], "mean", "n/a"))")
+    println("  ESS/sec (mean): $(get(performance["ess_per_sec"], "mean", "n/a"))")
+    println("  Divergences: $(divergences)")
+    println("  Divergence rate: $(divergence_rate === nothing ? "n/a" : round(divergence_rate, digits=6))")
+    println("  Mean leapfrog steps: $(get(performance["leapfrog"], "mean", "n/a"))")
+
+    return performance
+end
+

src/julia/validate_construct.jl

@@ -0,0 +1,450 @@
+#!/usr/bin/env julia
+#############################################################################
+## validate_construct.jl
+## Construct validity: Party family ordering and temporal stability
+##
+## Following Claassen (2019), this script validates:
+## 1. Party family ordering: Do family means follow theoretically expected orderings?
+## 2. Temporal stability: Flag parties with implausible position changes
+##
+## Uses ParlGov party family classifications (Döring & Manow 2024) via PartyFacts IDs.
+#############################################################################
+
+using CSV, DataFrames, Statistics, StatsBase, Dates, Printf
+
+# Family code → display name mapping
+const FAMILY_DISPLAY_NAMES = Dict(
+    "com" => "Communist/Far Left",
+    "eco" => "Green/Ecological",
+    "soc" => "Social Democratic",
+    "lib" => "Liberal",
+    "chr" => "Christian Democratic",
+    "con" => "Conservative",
+    "right" => "Radical Right"
+)
+
+# Substantive families (drop Specialist, Other, Agrarian — heterogeneous or ambiguous)
+const SUBSTANTIVE_FAMILIES = Set(["com", "eco", "soc", "lib", "chr", "con", "right"])
+
+# Expected orderings (theoretically motivated)
+# Economic: Communist < Social Democratic < Green < Christian Democratic < Conservative
+# (5-family core — Liberal position is ambiguous cross-nationally)
+const EXPECTED_ECONOMIC_ORDER = ["com", "soc", "eco", "chr", "con"]
+
+# Cultural: Green < Liberal < Social Democratic < Christian Democratic < Conservative < Radical Right
+const EXPECTED_GALTAN_ORDER = ["eco", "lib", "soc", "chr", "con", "right"]
+
+function load_model_output(base_dir::String=".")
+    """Load the most recent 2D model party positions output"""
+
+    position_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
+                           !endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(base_dir))
+    legacy_files = filter(f -> startswith(f, "party_positions_v1_") && endswith(f, ".csv"), readdir(base_dir))
+    append!(position_files, legacy_files)
+
+    if !isempty(position_files)
+        latest = sort(position_files)[end]
+        println("Loading model output: $latest")
+        return CSV.read(joinpath(base_dir, latest), DataFrame), latest
+    end
+
+    # Check output estimations directory
+    est_dir = joinpath(base_dir, "outputs", "estimations", "latest")
+    if isdir(est_dir)
+        est_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
+                           !endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(est_dir))
+        if !isempty(est_files)
+            latest = sort(est_files)[end]
+            println("Loading model output: outputs/estimations/latest/$latest")
+            return CSV.read(joinpath(est_dir, latest), DataFrame), latest
+        end
+    end
+
+    error("No party_positions_*.csv found. Run 02_post_estimation.jl first.")
+end
+
+function validate_party_families(model::DataFrame)
+    """Check whether party family means follow theoretically expected orderings"""
+
+    println("\n" * "="^60)
+    println("PARTY FAMILY ORDERING VALIDATION")
+    println("="^60)
+    println("\nUsing ParlGov family classifications (Döring & Manow 2024)")
+    println()
+
+    party_col = hasproperty(model, :party_id) ? :party_id : :party
+
+    # Load party families
+    families_df = CSV.read("data/party_families.csv", DataFrame)
+
+    # Join to model output
+    model_with_families = innerjoin(model, families_df, on=party_col => :partyfacts_id)
+
+    # Filter to substantive families
+    filter!(r -> r.family in SUBSTANTIVE_FAMILIES, model_with_families)
+
+    n_parties = length(unique(model_with_families[!, party_col]))
+    n_obs = nrow(model_with_families)
+    println("  Matched $n_parties parties ($n_obs party-years) across $(length(SUBSTANTIVE_FAMILIES)) families")
+    println()
+
+    # Compute family means
+    family_stats = combine(groupby(model_with_families, :family)) do df
+        DataFrame(
+            n_parties = length(unique(df[!, party_col])),
+            n_obs = nrow(df),
+            mean_economic = mean(df.economic_lr),
+            sd_economic = std(df.economic_lr),
+            mean_galtan = mean(df.galtan),
+            sd_galtan = std(df.galtan)
+        )
+    end
+
+    # Add display names
+    family_stats.family_name = [get(FAMILY_DISPLAY_NAMES, f, f) for f in family_stats.family]
+
+    # Sort by economic mean for display
+    sort!(family_stats, :mean_economic)
+
+    # Print table
+    @printf("  %-22s %7s %7s %10s %10s %10s %10s\n",
+        "Family", "Parties", "Obs", "Econ Mean", "Econ SD", "Cult Mean", "Cult SD")
+    println("  " * "-"^76)
+
+    for row in eachrow(family_stats)
+        @printf("  %-22s %7d %7d %10.3f %10.3f %10.3f %10.3f\n",
+            row.family_name, row.n_parties, row.n_obs,
+            row.mean_economic, row.sd_economic, row.mean_galtan, row.sd_galtan)
+    end
+
+    # Compute Spearman rank correlations for expected orderings
+    println()
+
+    # Economic ordering
+    econ_lookup = Dict(row.family => row.mean_economic for row in eachrow(family_stats))
+    econ_observed = [econ_lookup[f] for f in EXPECTED_ECONOMIC_ORDER if haskey(econ_lookup, f)]
+    econ_expected_ranks = collect(1:length(econ_observed))
+    econ_observed_ranks = ordinalrank(econ_observed)
+    rho_econ = corspearman(Float64.(econ_expected_ranks), Float64.(econ_observed_ranks))
+
+    println(@sprintf("  Economic ordering (5-family core): Spearman ρ = %.3f", rho_econ))
+    econ_families_used = [f for f in EXPECTED_ECONOMIC_ORDER if haskey(econ_lookup, f)]
+    econ_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in econ_families_used]
+    println("    Expected: ", join(econ_names, " < "))
+    observed_econ_order = econ_families_used[sortperm(econ_observed)]
+    observed_econ_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in observed_econ_order]
+    println("    Observed: ", join(observed_econ_names, " < "))
+
+    # Cultural ordering
+    galtan_lookup = Dict(row.family => row.mean_galtan for row in eachrow(family_stats))
+    galtan_observed = [galtan_lookup[f] for f in EXPECTED_GALTAN_ORDER if haskey(galtan_lookup, f)]
+    galtan_expected_ranks = collect(1:length(galtan_observed))
+    galtan_observed_ranks = ordinalrank(galtan_observed)
+    rho_galtan = corspearman(Float64.(galtan_expected_ranks), Float64.(galtan_observed_ranks))
+
+    println(@sprintf("  Cultural ordering (6-family): Spearman ρ = %.3f", rho_galtan))
+    galtan_families_used = [f for f in EXPECTED_GALTAN_ORDER if haskey(galtan_lookup, f)]
+    galtan_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in galtan_families_used]
+    println("    Expected: ", join(galtan_names, " < "))
+    observed_galtan_order = galtan_families_used[sortperm(galtan_observed)]
+    observed_galtan_names = [get(FAMILY_DISPLAY_NAMES, f, f) for f in observed_galtan_order]
+    println("    Observed: ", join(observed_galtan_names, " < "))
+
+    println()
+    println("-"^60)
+    if rho_econ >= 0.9 && rho_galtan >= 0.8
+        println("EXCELLENT: Family means follow expected orderings on both dimensions")
+    elseif rho_econ >= 0.7 && rho_galtan >= 0.7
+        println("GOOD: Family means broadly follow expected orderings")
+    else
+        println("CONCERN: Review family ordering results")
+    end
+
+    return family_stats, rho_econ, rho_galtan
+end
+
+function validate_temporal_stability(model::DataFrame)
+    """Check for implausible year-to-year position changes"""
+
+    println("\n" * "="^60)
+    println("TEMPORAL STABILITY VALIDATION")
+    println("="^60)
+    println("\nFlagging parties with >0.10 change per year")
+    println()
+
+    party_col = hasproperty(model, :party_id) ? :party_id : :party
+
+    # Compute year-to-year changes within each party
+    sort!(model, [party_col, :year])
+
+    unstable_parties = []
+
+    for party_df in groupby(model, party_col)
+        if nrow(party_df) < 2
+            continue
+        end
+
+        party_id = party_df[1, party_col]
+        country = party_df[1, :country]
+
+        # Compute differences
+        for dim in [:economic_lr, :galtan]
+            vals = party_df[!, dim]
+            years = party_df.year
+
+            for i in 2:length(vals)
+                diff = abs(vals[i] - vals[i-1])
+                year_gap = years[i] - years[i-1]
+
+                # Normalize by year gap (handle multi-year gaps)
+                annual_change = diff / max(year_gap, 1)
+
+                if annual_change > 0.10
+                    push!(unstable_parties, (
+                        party_id = party_id,
+                        country = country,
+                        dimension = string(dim),
+                        year_from = years[i-1],
+                        year_to = years[i],
+                        val_from = vals[i-1],
+                        val_to = vals[i],
+                        change = diff,
+                        annual_change = annual_change
+                    ))
+                end
+            end
+        end
+    end
+
+    if isempty(unstable_parties)
+        println("  No parties with >0.10 annual change found")
+        println("  EXCELLENT: Positions are temporally stable")
+        return DataFrame()
+    end
+
+    unstable_df = DataFrame(unstable_parties)
+    sort!(unstable_df, :annual_change, rev=true)
+
+    println("  Found $(nrow(unstable_df)) instances of rapid change:")
+    println()
+    @printf("  %-8s %-8s %-12s %-10s %-10s %8s\n",
+        "Party", "Country", "Dimension", "Years", "Change", "Annual")
+    println("  " * "-"^60)
+
+    for row in eachrow(unstable_df[1:min(20, nrow(unstable_df)), :])
+        @printf("  %-8d %-8s %-12s %d->%d %8.3f %8.3f\n",
+            row.party_id, row.country, row.dimension,
+            row.year_from, row.year_to, row.change, row.annual_change)
+    end
+
+    if nrow(unstable_df) > 20
+        println("  ... and $(nrow(unstable_df) - 20) more")
+    end
+
+    println()
+    println("-"^60)
+    n_parties = length(unique(unstable_df.party_id))
+    n_total = length(unique(model[!, party_col]))
+    println(@sprintf("Unstable parties: %d/%d (%.1f%%)", n_parties, n_total, 100*n_parties/n_total))
+
+    return unstable_df
+end
+
+function validate_position_distributions(model::DataFrame)
+    """Check overall distribution of positions makes sense"""
+
+    println("\n" * "="^60)
+    println("POSITION DISTRIBUTION VALIDATION")
+    println("="^60)
+    println("\nSummary statistics for model estimates")
+    println()
+
+    for dim in [:economic_lr, :galtan]
+        if !hasproperty(model, dim)
+            continue
+        end
+
+        vals = model[!, dim]
+        println("$dim:")
+        println(@sprintf("  Mean:   %.3f (should be ~0.50)", mean(vals)))
+        println(@sprintf("  Median: %.3f (should be ~0.50)", median(vals)))
+        println(@sprintf("  SD:     %.3f (should be ~0.15)", std(vals)))
+        println(@sprintf("  Min:    %.3f", minimum(vals)))
+        println(@sprintf("  Max:    %.3f", maximum(vals)))
+        println(@sprintf("  Q25:    %.3f", quantile(vals, 0.25)))
+        println(@sprintf("  Q75:    %.3f", quantile(vals, 0.75)))
+        println()
+    end
+
+    # Check for extreme values
+    println("Extreme positions (< 0.10 or > 0.90):")
+
+    party_col = hasproperty(model, :party_id) ? :party_id : :party
+
+    for dim in [:economic_lr, :galtan]
+        if !hasproperty(model, dim)
+            continue
+        end
+
+        extreme = filter(row -> row[dim] < 0.10 || row[dim] > 0.90, model)
+        n_extreme = nrow(extreme)
+        pct_extreme = 100 * n_extreme / nrow(model)
+
+        println(@sprintf("  %s: %d (%.1f%%)", dim, n_extreme, pct_extreme))
+
+        if n_extreme > 0 && n_extreme <= 10
+            for row in eachrow(extreme[1:min(5, nrow(extreme)), :])
+                println(@sprintf("    Party %d (%s) %d: %.3f",
+                    row[party_col], row.country, row.year, row[dim]))
+            end
+        end
+    end
+end
+
+function validate_country_patterns(model::DataFrame)
+    """Check country-level patterns make sense"""
+
+    println("\n" * "="^60)
+    println("COUNTRY-LEVEL VALIDATION")
+    println("="^60)
+    println("\nMean positions by country (should vary but not wildly)")
+    println()
+
+    country_stats = combine(groupby(model, :country)) do df
+        DataFrame(
+            n_parties = length(unique(hasproperty(df, :party_id) ? df.party_id : df.party)),
+            n_obs = nrow(df),
+            mean_econ = mean(df.economic_lr),
+            mean_galtan = mean(df.galtan),
+            sd_econ = std(df.economic_lr),
+            sd_galtan = std(df.galtan)
+        )
+    end
+
+    sort!(country_stats, :n_obs, rev=true)
+
+    @printf("%-4s %6s %6s %8s %8s %8s %8s\n",
+        "CC", "Parties", "N", "Econ", "SD", "Galtan", "SD")
+    println("-"^60)
+
+    for row in eachrow(country_stats[1:min(20, nrow(country_stats)), :])
+        @printf("%-4s %6d %6d %8.3f %8.3f %8.3f %8.3f\n",
+            row.country, row.n_parties, row.n_obs,
+            row.mean_econ, row.sd_econ, row.mean_galtan, row.sd_galtan)
+    end
+
+    # Flag countries with unusual patterns
+    println("\nCountries with unusual patterns:")
+    unusual = filter(row -> row.mean_econ < 0.35 || row.mean_econ > 0.65 ||
+                           row.mean_galtan < 0.35 || row.mean_galtan > 0.65, country_stats)
+
+    if nrow(unusual) == 0
+        println("  None - all countries have balanced party systems")
+    else
+        for row in eachrow(unusual)
+            issues = String[]
+            if row.mean_econ < 0.35
+                push!(issues, "left-skewed economy")
+            elseif row.mean_econ > 0.65
+                push!(issues, "right-skewed economy")
+            end
+            if row.mean_galtan < 0.35
+                push!(issues, "cultural-skewed")
+            elseif row.mean_galtan > 0.65
+                push!(issues, "TAN-skewed")
+            end
+            println("  $(row.country): $(join(issues, ", "))")
+        end
+    end
+
+    return country_stats
+end
+
+function save_construct_results(families::DataFrame, unstable::DataFrame,
+                                countries::DataFrame, output_dir::String="outputs/checks")
+    """Save construct validation results"""
+
+    if !isdir(output_dir)
+        mkpath(output_dir)
+    end
+
+    timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")
+
+    if nrow(families) > 0
+        families_file = joinpath(output_dir, "construct_families_$timestamp.csv")
+        CSV.write(families_file, families)
+        println("\nSaved: $families_file")
+    end
+
+    if nrow(unstable) > 0
+        unstable_file = joinpath(output_dir, "construct_unstable_$timestamp.csv")
+        CSV.write(unstable_file, unstable)
+        println("Saved: $unstable_file")
+    end
+
+    if nrow(countries) > 0
+        country_file = joinpath(output_dir, "construct_countries_$timestamp.csv")
+        CSV.write(country_file, countries)
+        println("Saved: $country_file")
+    end
+end
+
+
+# Main execution
+function main()
+    println("="^60)
+    println("CONSTRUCT VALIDITY: Face Validity Checks")
+    println("="^60)
+    println("Checking if model estimates match expectations")
+    println()
+
+    # Load data
+    model, model_file = load_model_output()
+
+    println("\nModel output:")
+    println("  Rows: $(nrow(model))")
+    println("  Columns: $(names(model))")
+
+    # Run validations
+    families, rho_econ, rho_galtan = validate_party_families(model)
+    unstable = validate_temporal_stability(model)
+    validate_position_distributions(model)
+    countries = validate_country_patterns(model)
+
+    # Save results
+    save_construct_results(families, unstable, countries)
+
+    # Summary
+    println("\n" * "="^60)
+    println("CONSTRUCT VALIDITY SUMMARY")
+    println("="^60)
+
+    println(@sprintf("  Party family ordering:"))
+    println(@sprintf("    Economic (5-family): Spearman ρ = %.3f", rho_econ))
+    println(@sprintf("    Cultural (6-family):  Spearman ρ = %.3f", rho_galtan))
+
+    n_unstable = nrow(unstable) > 0 ? length(unique(unstable.party_id)) : 0
+    party_col = hasproperty(model, :party_id) ? :party_id : :party
+    n_total = length(unique(model[!, party_col]))
+    println(@sprintf("  Temporal stability: %d/%d parties stable (>0.10/yr threshold)",
+        n_total - n_unstable, n_total))
+
+    if rho_econ >= 0.9 && rho_galtan >= 0.8 && n_unstable < 0.1 * n_total
+        println("\n  EXCELLENT: Model has good construct validity")
+    elseif rho_econ >= 0.7 && rho_galtan >= 0.7
+        println("\n  GOOD: Model has reasonable construct validity")
+    else
+        println("\n  CONCERN: Review family ordering results")
+    end
+
+
+    println("\n" * "="^60)
+    println("VALIDATION COMPLETE")
+    println("="^60)
+
+    return (families=families, unstable=unstable, countries=countries)
+end
+
+if abspath(PROGRAM_FILE) == @__FILE__
+    main()
+end

src/julia/validate_convergent.jl

@@ -0,0 +1,533 @@
+#!/usr/bin/env julia
+#############################################################################
+## validate_convergent.jl
+## Convergent validity: Compare model estimates to external expert surveys
+##
+## Following Claassen (2019), this script computes:
+## - Pearson/Spearman correlations between model and expert estimates
+## - Fisher z-transformation for correlation confidence intervals
+## - Mean Absolute Error (MAE) and Root Mean Square Error (RMSE)
+## - Breakdown by survey project and decade
+##
+## Target: r > 0.8 with CHES (Claassen achieved 0.50-0.57)
+#############################################################################
+
+using CSV, DataFrames, Statistics, StatsBase, Dates, Printf, JSON
+
+# Fisher z-transformation for correlation confidence intervals
+fisher_z(r) = 0.5 * log((1 + r) / (1 - r))
+fisher_z_inv(z) = (exp(2z) - 1) / (exp(2z) + 1)
+
+function correlation_ci(r, n; alpha=0.05)
+    """Calculate confidence interval for correlation using Fisher z-transformation"""
+    if n < 4
+        return (lower=NaN, upper=NaN)
+    end
+    z = fisher_z(r)
+    se = 1 / sqrt(n - 3)
+    z_crit = 1.96  # For 95% CI
+    z_lower = z - z_crit * se
+    z_upper = z + z_crit * se
+    return (lower=fisher_z_inv(z_lower), upper=fisher_z_inv(z_upper))
+end
+
+function load_model_output(base_dir::String=".")
+    """Load the most recent 2D model party positions output"""
+
+    # First check for post_estimation output in root (current name, with legacy fallback)
+    position_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
+                           !endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(base_dir))
+    legacy_files = filter(f -> startswith(f, "party_positions_v1_") && endswith(f, ".csv"), readdir(base_dir))
+    append!(position_files, legacy_files)
+
+    if !isempty(position_files)
+        latest = sort(position_files)[end]
+        println("Loading model output: $latest")
+        return CSV.read(joinpath(base_dir, latest), DataFrame), latest
+    end
+
+    # Check current pipeline output directory, with legacy estimations/ fallback
+    for (label, est_dir) in [
+        ("outputs/estimations/latest", joinpath(base_dir, "outputs", "estimations", "latest")),
+        ("estimations", joinpath(base_dir, "estimations")),
+    ]
+        if isdir(est_dir)
+            est_files = filter(f -> startswith(f, "party_positions_") && endswith(f, ".csv") &&
+                               !endswith(f, "_metadata.txt") && !endswith(f, "_tables.tex"), readdir(est_dir))
+            if !isempty(est_files)
+                latest = sort(est_files)[end]
+                println("Loading model output: $label/$latest")
+                return CSV.read(joinpath(est_dir, latest), DataFrame), latest
+            end
+        end
+    end
+
+    error("No party_positions_*.csv found. Run 02_post_estimation.jl first.")
+end
+
+function load_expert_data(base_dir::String=".")
+    """Load expert survey data files"""
+
+    data_dir = isfile(joinpath(base_dir, "expert.csv")) ? base_dir : joinpath(base_dir, "data")
+
+    # Load dimension-specific expert data
+    expert_file = joinpath(data_dir, "expert.csv")
+    if !isfile(expert_file)
+        error("expert.csv not found in $base_dir or $(joinpath(base_dir, "data"))")
+    end
+
+    println("Loading expert.csv...")
+    expert = CSV.read(expert_file, DataFrame)
+    println("  Rows: $(nrow(expert))")
+    println("  Variables: $(unique(expert.var))")
+
+    # Load L-R data
+    lr_file = joinpath(data_dir, "lr_data.csv")
+    if !isfile(lr_file)
+        error("lr_data.csv not found in $base_dir or $(joinpath(base_dir, "data"))")
+    end
+
+    println("Loading lr_data.csv...")
+    lr_data = CSV.read(lr_file, DataFrame)
+    println("  Rows: $(nrow(lr_data))")
+    println("  Variables: $(unique(lr_data.var))")
+
+    return expert, lr_data
+end
+
+function validate_economic_lr(model::DataFrame, expert::DataFrame)
+    """Validate economic_lr against CHES/V-Party/POPPA/GPS lrecon"""
+
+    println("\n" * "="^60)
+    println("CONVERGENT VALIDITY: economic_lr")
+    println("="^60)
+
+    # Filter expert data for economic dimension
+    econ_vars = filter(v -> startswith(v, "lrecon_"), unique(expert.var))
+    econ_expert = filter(row -> row.var in econ_vars, expert)
+
+    println("\nExpert data variables: $(econ_vars)")
+    println("Expert observations: $(nrow(econ_expert))")
+
+    # Merge with model output
+    # Model has party_id column (from segment-based), expert has party column
+    if hasproperty(model, :party_id)
+        model_merge = select(model, :party_id => :party, :year, :economic_lr, :economic_lr_se)
+    else
+        model_merge = select(model, :party, :year, :economic_lr, :economic_lr_se)
+    end
+
+    merged = innerjoin(econ_expert, model_merge, on=[:party, :year])
+
+    println("Merged observations: $(nrow(merged))")
+
+    if nrow(merged) < 10
+        println("WARNING: Too few observations for meaningful validation")
+        return nothing
+    end
+
+    # Compute overall correlation
+    r_pearson = cor(merged.val, merged.economic_lr)
+    r_spearman = corspearman(merged.val, merged.economic_lr)
+    mae = mean(abs.(merged.val .- merged.economic_lr))
+    rmse = sqrt(mean((merged.val .- merged.economic_lr).^2))
+
+    ci = correlation_ci(r_pearson, nrow(merged))
+
+    println("\n--- Overall Statistics ---")
+    println(@sprintf("  Pearson r:  %.4f [%.4f, %.4f]", r_pearson, ci.lower, ci.upper))
+    println(@sprintf("  Spearman r: %.4f", r_spearman))
+    println(@sprintf("  MAE:        %.4f", mae))
+    println(@sprintf("  RMSE:       %.4f", rmse))
+    println(@sprintf("  N:          %d", nrow(merged)))
+
+    # Breakdown by project
+    println("\n--- By Project ---")
+    by_project = combine(groupby(merged, :project)) do df
+        n = nrow(df)
+        if n < 3
+            return DataFrame(n=n, r_pearson=NaN, mae=NaN)
+        end
+        DataFrame(
+            n = n,
+            r_pearson = cor(df.val, df.economic_lr),
+            r_spearman = corspearman(df.val, df.economic_lr),
+            mae = mean(abs.(df.val .- df.economic_lr)),
+            rmse = sqrt(mean((df.val .- df.economic_lr).^2))
+        )
+    end
+
+    for row in eachrow(sort(by_project, :n, rev=true))
+        if !isnan(row.r_pearson)
+            println(@sprintf("  %-10s: r=%.3f, MAE=%.3f, n=%d",
+                row.project, row.r_pearson, row.mae, row.n))
+        end
+    end
+
+    # Breakdown by decade
+    println("\n--- By Decade ---")
+    merged.decade = div.(merged.year, 10) .* 10
+    by_decade = combine(groupby(merged, :decade)) do df
+        n = nrow(df)
+        if n < 3
+            return DataFrame(n=n, r_pearson=NaN, mae=NaN)
+        end
+        DataFrame(
+            n = n,
+            r_pearson = cor(df.val, df.economic_lr),
+            mae = mean(abs.(df.val .- df.economic_lr))
+        )
+    end
+
+    for row in eachrow(sort(by_decade, :decade))
+        if !isnan(row.r_pearson)
+            println(@sprintf("  %ds: r=%.3f, MAE=%.3f, n=%d",
+                row.decade, row.r_pearson, row.mae, row.n))
+        end
+    end
+
+    return (
+        dimension = "economic_lr",
+        r_pearson = r_pearson,
+        r_spearman = r_spearman,
+        ci_lower = ci.lower,
+        ci_upper = ci.upper,
+        mae = mae,
+        rmse = rmse,
+        n = nrow(merged),
+        by_project = by_project,
+        by_decade = by_decade
+    )
+end
+
+function validate_galtan(model::DataFrame, expert::DataFrame)
+    """Validate galtan against CHES galtan and V-Party/GPS libcons"""
+
+    println("\n" * "="^60)
+    println("CONVERGENT VALIDITY: galtan")
+    println("="^60)
+
+    # Filter expert data for GAL-TAN dimension
+    galtan_vars = filter(v -> occursin("galtan", v) || occursin("libcon", v), unique(expert.var))
+    galtan_expert = filter(row -> row.var in galtan_vars, expert)
+
+    println("\nExpert data variables: $(galtan_vars)")
+    println("Expert observations: $(nrow(galtan_expert))")
+
+    # Merge with model output
+    if hasproperty(model, :party_id)
+        model_merge = select(model, :party_id => :party, :year, :galtan, :galtan_se)
+    else
+        model_merge = select(model, :party, :year, :galtan, :galtan_se)
+    end
+
+    merged = innerjoin(galtan_expert, model_merge, on=[:party, :year])
+
+    println("Merged observations: $(nrow(merged))")
+
+    if nrow(merged) < 10
+        println("WARNING: Too few observations for meaningful validation")
+        return nothing
+    end
+
+    # Compute overall correlation
+    r_pearson = cor(merged.val, merged.galtan)
+    r_spearman = corspearman(merged.val, merged.galtan)
+    mae = mean(abs.(merged.val .- merged.galtan))
+    rmse = sqrt(mean((merged.val .- merged.galtan).^2))
+
+    ci = correlation_ci(r_pearson, nrow(merged))
+
+    println("\n--- Overall Statistics ---")
+    println(@sprintf("  Pearson r:  %.4f [%.4f, %.4f]", r_pearson, ci.lower, ci.upper))
+    println(@sprintf("  Spearman r: %.4f", r_spearman))
+    println(@sprintf("  MAE:        %.4f", mae))
+    println(@sprintf("  RMSE:       %.4f", rmse))
+    println(@sprintf("  N:          %d", nrow(merged)))
+
+    # Breakdown by project
+    println("\n--- By Project ---")
+    by_project = combine(groupby(merged, :project)) do df
+        n = nrow(df)
+        if n < 3
+            return DataFrame(n=n, r_pearson=NaN, mae=NaN)
+        end
+        DataFrame(
+            n = n,
+            r_pearson = cor(df.val, df.galtan),
+            r_spearman = corspearman(df.val, df.galtan),
+            mae = mean(abs.(df.val .- df.galtan)),
+            rmse = sqrt(mean((df.val .- df.galtan).^2))
+        )
+    end
+
+    for row in eachrow(sort(by_project, :n, rev=true))
+        if !isnan(row.r_pearson)
+            println(@sprintf("  %-10s: r=%.3f, MAE=%.3f, n=%d",
+                row.project, row.r_pearson, row.mae, row.n))
+        end
+    end
+
+    # Breakdown by decade
+    println("\n--- By Decade ---")
+    merged.decade = div.(merged.year, 10) .* 10
+    by_decade = combine(groupby(merged, :decade)) do df
+        n = nrow(df)
+        if n < 3
+            return DataFrame(n=n, r_pearson=NaN, mae=NaN)
+        end
+        DataFrame(
+            n = n,
+            r_pearson = cor(df.val, df.galtan),
+            mae = mean(abs.(df.val .- df.galtan))
+        )
+    end
+
+    for row in eachrow(sort(by_decade, :decade))
+        if !isnan(row.r_pearson)
+            println(@sprintf("  %ds: r=%.3f, MAE=%.3f, n=%d",
+                row.decade, row.r_pearson, row.mae, row.n))
+        end
+    end
+
+    return (
+        dimension = "galtan",
+        r_pearson = r_pearson,
+        r_spearman = r_spearman,
+        ci_lower = ci.lower,
+        ci_upper = ci.upper,
+        mae = mae,
+        rmse = rmse,
+        n = nrow(merged),
+        by_project = by_project,
+        by_decade = by_decade
+    )
+end
+
+function validate_discriminant(model::DataFrame, expert::DataFrame)
+    """Compute cross-dimension correlations for discriminant validity (Campbell & Fiske 1959 MTMM)"""
+
+    println("\n" * "="^60)
+    println("DISCRIMINANT VALIDITY: Cross-dimension correlations")
+    println("="^60)
+    println("\nCampbell & Fiske (1959) MTMM framework:")
+    println("  Convergent: same dimension, different method → HIGH")
+    println("  Discriminant: different dimension, different method → LOW")
+    println()
+
+    # Get party column
+    if hasproperty(model, :party_id)
+        model_econ = select(model, :party_id => :party, :year, :economic_lr)
+        model_gal = select(model, :party_id => :party, :year, :galtan)
+    else
+        model_econ = select(model, :party, :year, :economic_lr)
+        model_gal = select(model, :party, :year, :galtan)
+    end
+
+    results = []
+
+    # 1. Expert economic vs Model economic (convergent - already computed, include for matrix)
+    econ_vars = filter(v -> startswith(v, "lrecon_"), unique(expert.var))
+    econ_expert = filter(row -> row.var in econ_vars, expert)
+    merged_ee = innerjoin(econ_expert, model_econ, on=[:party, :year])
+    if nrow(merged_ee) >= 10
+        r = cor(merged_ee.val, merged_ee.economic_lr)
+        push!(results, (model_dim="economic_lr", expert_dim="economic",
+                        r_pearson=r, r_spearman=corspearman(merged_ee.val, merged_ee.economic_lr),
+                        n=nrow(merged_ee), type="convergent"))
+        @printf("  Model Economic  × Expert Economic:  r = %.3f (convergent, n=%d)\n", r, nrow(merged_ee))
+    end
+
+    # 2. Expert economic vs Model galtan (discriminant)
+    merged_eg = innerjoin(econ_expert, model_gal, on=[:party, :year])
+    if nrow(merged_eg) >= 10
+        r = cor(merged_eg.val, merged_eg.galtan)
+        push!(results, (model_dim="galtan", expert_dim="economic",
+                        r_pearson=r, r_spearman=corspearman(merged_eg.val, merged_eg.galtan),
+                        n=nrow(merged_eg), type="discriminant"))
+        @printf("  Model GAL-TAN   × Expert Economic:  r = %.3f (discriminant, n=%d)\n", r, nrow(merged_eg))
+    end
+
+    # 3. Expert galtan vs Model galtan (convergent - already computed, include for matrix)
+    galtan_vars = filter(v -> occursin("galtan", v) || occursin("libcon", v), unique(expert.var))
+    galtan_expert = filter(row -> row.var in galtan_vars, expert)
+    merged_gg = innerjoin(galtan_expert, model_gal, on=[:party, :year])
+    if nrow(merged_gg) >= 10
+        r = cor(merged_gg.val, merged_gg.galtan)
+        push!(results, (model_dim="galtan", expert_dim="galtan",
+                        r_pearson=r, r_spearman=corspearman(merged_gg.val, merged_gg.galtan),
+                        n=nrow(merged_gg), type="convergent"))
+        @printf("  Model GAL-TAN   × Expert GAL-TAN:   r = %.3f (convergent, n=%d)\n", r, nrow(merged_gg))
+    end
+
+    # 4. Expert galtan vs Model economic (discriminant)
+    merged_ge = innerjoin(galtan_expert, model_econ, on=[:party, :year])
+    if nrow(merged_ge) >= 10
+        r = cor(merged_ge.val, merged_ge.economic_lr)
+        push!(results, (model_dim="economic_lr", expert_dim="galtan",
+                        r_pearson=r, r_spearman=corspearman(merged_ge.val, merged_ge.economic_lr),
+                        n=nrow(merged_ge), type="discriminant"))
+        @printf("  Model Economic  × Expert GAL-TAN:   r = %.3f (discriminant, n=%d)\n", r, nrow(merged_ge))
+    end
+
+    println()
+    println("MTMM Matrix:")
+    println("                    Expert Economic    Expert GAL-TAN")
+    for r in results
+        if r.model_dim == "economic_lr" && r.expert_dim == "economic"
+            @printf("  Model Economic:      %.3f            ", r.r_pearson)
+        end
+    end
+    for r in results
+        if r.model_dim == "economic_lr" && r.expert_dim == "galtan"
+            @printf("%.3f\n", r.r_pearson)
+        end
+    end
+    for r in results
+        if r.model_dim == "galtan" && r.expert_dim == "economic"
+            @printf("  Model GAL-TAN:       %.3f            ", r.r_pearson)
+        end
+    end
+    for r in results
+        if r.model_dim == "galtan" && r.expert_dim == "galtan"
+            @printf("%.3f\n", r.r_pearson)
+        end
+    end
+
+    return DataFrame(results)
+end
+
+function save_validation_results(results::Vector, output_dir::String="validation";
+                                  discriminant::Union{DataFrame, Nothing}=nothing)
+    """Save validation results to CSV files"""
+
+    if !isdir(output_dir)
+        mkpath(output_dir)
+    end
+
+    timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")
+
+    # Summary table
+    summary_rows = []
+    for r in results
+        if r !== nothing
+            push!(summary_rows, (
+                dimension = r.dimension,
+                r_pearson = r.r_pearson,
+                r_spearman = r.r_spearman,
+                ci_lower = r.ci_lower,
+                ci_upper = r.ci_upper,
+                mae = r.mae,
+                rmse = r.rmse,
+                n = r.n
+            ))
+        end
+    end
+
+    if !isempty(summary_rows)
+        summary_df = DataFrame(summary_rows)
+        summary_file = joinpath(output_dir, "convergent_summary_$timestamp.csv")
+        CSV.write(summary_file, summary_df)
+        println("\nSaved: $summary_file")
+    end
+
+    # By-project tables
+    for r in results
+        if r !== nothing && hasproperty(r, :by_project) && r.by_project !== nothing
+            project_file = joinpath(output_dir, "convergent_$(r.dimension)_by_project_$timestamp.csv")
+            CSV.write(project_file, r.by_project)
+            println("Saved: $project_file")
+        end
+    end
+
+    # By-decade tables
+    for r in results
+        if r !== nothing && hasproperty(r, :by_decade) && r.by_decade !== nothing
+            decade_file = joinpath(output_dir, "convergent_$(r.dimension)_by_decade_$timestamp.csv")
+            CSV.write(decade_file, r.by_decade)
+            println("Saved: $decade_file")
+        end
+    end
+
+    # Discriminant validity table
+    if discriminant !== nothing && nrow(discriminant) > 0
+        disc_file = joinpath(output_dir, "discriminant_summary_$timestamp.csv")
+        CSV.write(disc_file, discriminant)
+        println("Saved: $disc_file")
+    end
+
+    return summary_rows
+end
+
+function print_claassen_comparison(results::Vector)
+    """Print comparison with Claassen (2019) benchmarks"""
+
+    println("\n" * "="^60)
+    println("COMPARISON WITH CLAASSEN (2019) BENCHMARKS")
+    println("="^60)
+
+    println("\nClaassen's results (mood estimates vs survey data):")
+    println("  Pearson r: 0.50-0.57")
+    println("  MAE: ~0.06 (6 pp on 0-1 scale)")
+    println()
+
+    println("Our target (party positions, should be HIGHER than mood):")
+    println("  Pearson r > 0.80 with expert surveys")
+    println("  MAE < 0.15 (reasonable measurement error)")
+    println()
+
+    println("-"^60)
+    @printf("%-15s %8s %8s %8s %8s\n", "Dimension", "r", "Target", "MAE", "Status")
+    println("-"^60)
+
+    for r in results
+        if r !== nothing
+            status = r.r_pearson > 0.80 ? "PASS" : (r.r_pearson > 0.70 ? "OK" : "LOW")
+            @printf("%-15s %8.3f %8s %8.3f %8s\n",
+                r.dimension, r.r_pearson, "> 0.80", r.mae, status)
+        end
+    end
+    println("-"^60)
+end
+
+# Main execution
+function main()
+    println("="^60)
+    println("CONVERGENT VALIDITY: Model vs Expert Surveys")
+    println("="^60)
+    println("Following Claassen (2019) validation framework")
+    println()
+
+    # Load data
+    model, model_file = load_model_output()
+    expert, _ = load_expert_data()
+
+    println("\nModel output:")
+    println("  Rows: $(nrow(model))")
+    println("  Columns: $(names(model))")
+
+    # Run validations
+    results = []
+
+    push!(results, validate_economic_lr(model, expert))
+    push!(results, validate_galtan(model, expert))
+
+    # Run discriminant validity
+    discriminant = validate_discriminant(model, expert)
+
+    # Save results
+    save_validation_results(results; discriminant=discriminant)
+
+    # Print Claassen comparison
+    print_claassen_comparison(results)
+
+    println("\n" * "="^60)
+    println("VALIDATION COMPLETE")
+    println("="^60)
+
+    return results
+end
+
+if abspath(PROGRAM_FILE) == @__FILE__
+    main()
+end

src/julia/validate_external.jl

@@ -0,0 +1,375 @@
+#!/usr/bin/env julia
+#############################################################################
+## validate_external.jl
+## Out-of-sample validation via held-out expert observations
+##
+## Design:
+##   - Text data stays 100% intact (same parties, segments, indices)
+##   - 20% of expert/LR observations held out (stratified by source)
+##   - Expert-only parties (no text data) are never held out
+##   - Model trains on 80% expert + 100% text
+##   - Held-out expert ratings compared to model predictions
+##
+## Usage:
+##   julia scripts/validate_external.jl prepare
+##   julia 01_run_model.jl --data-dir validation/external_split/
+##   julia 02_post_estimation.jl  # on training run
+##   julia scripts/validate_external.jl compute <model_positions.csv>
+#############################################################################
+
+using CSV, DataFrames, Statistics, Random, Dates, Printf
+
+const HOLDOUT_FRAC = 0.20
+const SEED = 42
+
+# =========================================================================
+# STEP 1: Prepare train/test split
+# =========================================================================
+
+function prepare_holdout_data(base_dir::String=".")
+    println("="^70)
+    println("PREPARING OUT-OF-SAMPLE VALIDATION SPLIT")
+    println("="^70)
+    println()
+
+    # Load data
+    text_data = CSV.read(joinpath(base_dir, "text_data.csv"), DataFrame)
+    expert = CSV.read(joinpath(base_dir, "expert.csv"), DataFrame)
+    lr_data = CSV.read(joinpath(base_dir, "lr_data.csv"), DataFrame)
+
+    println("Full dataset:")
+    println("  text_data: $(nrow(text_data)) rows, $(length(unique(text_data.party))) parties")
+    println("  expert:    $(nrow(expert)) rows, $(length(unique(expert.party))) parties")
+    println("  lr_data:   $(nrow(lr_data)) rows, $(length(unique(lr_data.party))) parties")
+
+    # Identify expert-only parties (no text data) — these are NEVER held out
+    text_parties = Set(unique(text_data.party))
+    expert_only_parties = Set(p for p in unique(vcat(expert.party, lr_data.party))
+                              if !(p in text_parties))
+
+    println()
+    println("Expert-only parties (protected from holdout): $(length(expert_only_parties))")
+
+    # Split expert data: stratified by source variable
+    # Safeguard: ensure each party keeps at least one observation in training
+    Random.seed!(SEED)
+    expert.row_id = 1:nrow(expert)
+    expert.is_holdout = falses(nrow(expert))
+
+    for var_group in groupby(expert, :var)
+        var_name = first(var_group.var)
+        eligible = findall(row -> !(row.party in expert_only_parties), eachrow(var_group))
+
+        n_holdout = round(Int, length(eligible) * HOLDOUT_FRAC)
+        holdout_candidates = shuffle(eligible)
+
+        # Track per-party counts to ensure at least 1 stays in training
+        party_train_count = Dict{Int, Int}()
+        for idx in eligible
+            p = var_group.party[idx]
+            party_train_count[p] = get(party_train_count, p, 0) + 1
+        end
+
+        n_held = 0
+        for idx in holdout_candidates
+            n_held >= n_holdout && break
+            p = var_group.party[idx]
+            if party_train_count[p] > 1  # keep at least 1 in training
+                row_id = var_group.row_id[idx]
+                expert.is_holdout[row_id] = true
+                party_train_count[p] -= 1
+                n_held += 1
+            end
+        end
+    end
+
+    # Split LR data: stratified by source variable (same safeguard)
+    lr_data.row_id = 1:nrow(lr_data)
+    lr_data.is_holdout = falses(nrow(lr_data))
+
+    for var_group in groupby(lr_data, :var)
+        var_name = first(var_group.var)
+        eligible = findall(row -> !(row.party in expert_only_parties), eachrow(var_group))
+
+        n_holdout = round(Int, length(eligible) * HOLDOUT_FRAC)
+        holdout_candidates = shuffle(eligible)
+
+        party_train_count = Dict{Int, Int}()
+        for idx in eligible
+            p = var_group.party[idx]
+            party_train_count[p] = get(party_train_count, p, 0) + 1
+        end
+
+        n_held = 0
+        for idx in holdout_candidates
+            n_held >= n_holdout && break
+            p = var_group.party[idx]
+            if party_train_count[p] > 1
+                row_id = var_group.row_id[idx]
+                lr_data.is_holdout[row_id] = true
+                party_train_count[p] -= 1
+                n_held += 1
+            end
+        end
+    end
+
+    # Create train/test splits
+    expert_train = expert[.!expert.is_holdout, Not([:row_id, :is_holdout])]
+    expert_test = expert[expert.is_holdout, Not([:row_id, :is_holdout])]
+    lr_train = lr_data[.!lr_data.is_holdout, Not([:row_id, :is_holdout])]
+    lr_test = lr_data[lr_data.is_holdout, Not([:row_id, :is_holdout])]
+
+    # Report split
+    println()
+    println("Split summary ($(round(100*HOLDOUT_FRAC))% holdout):")
+    println("  expert train: $(nrow(expert_train)) rows ($(round(100*nrow(expert_train)/nrow(expert), digits=1))%)")
+    println("  expert test:  $(nrow(expert_test)) rows ($(round(100*nrow(expert_test)/nrow(expert), digits=1))%)")
+    println("  lr train:     $(nrow(lr_train)) rows ($(round(100*nrow(lr_train)/nrow(lr_data), digits=1))%)")
+    println("  lr test:      $(nrow(lr_test)) rows ($(round(100*nrow(lr_test)/nrow(lr_data), digits=1))%)")
+
+    # Report per-source breakdown
+    println()
+    println("Per-source breakdown (expert):")
+    for var_name in sort(unique(expert.var))
+        n_full = count(expert.var .== var_name)
+        n_test = count(expert_test.var .== var_name)
+        println("  $var_name: $(n_full - n_test) train / $n_test test")
+    end
+    println()
+    println("Per-source breakdown (LR):")
+    for var_name in sort(unique(lr_data.var))
+        n_full = count(lr_data.var .== var_name)
+        n_test = count(lr_test.var .== var_name)
+        println("  $var_name: $(n_full - n_test) train / $n_test test")
+    end
+
+    # Verify: training set has same parties as full set
+    train_parties_expert = Set(unique(expert_train.party))
+    train_parties_lr = Set(unique(lr_train.party))
+    full_parties_expert = Set(unique(expert.party))
+    full_parties_lr = Set(unique(lr_data.party))
+
+    lost_expert = setdiff(full_parties_expert, train_parties_expert)
+    lost_lr = setdiff(full_parties_lr, train_parties_lr)
+
+    println()
+    if isempty(lost_expert) && isempty(lost_lr)
+        println("✓ No parties lost from training set")
+    else
+        println("⚠ Parties lost from expert training: $(length(lost_expert))")
+        println("⚠ Parties lost from LR training: $(length(lost_lr))")
+    end
+
+    # Save to output directory
+    # Files use standard names so 01_run_model.jl can load with --data-dir
+    output_dir = joinpath(base_dir, "validation", "external_split")
+    mkpath(output_dir)
+
+    # Training files (standard names for model loading)
+    CSV.write(joinpath(output_dir, "text_data.csv"), text_data)  # UNCHANGED
+    CSV.write(joinpath(output_dir, "expert.csv"), expert_train)
+    CSV.write(joinpath(output_dir, "lr_data.csv"), lr_train)
+
+    # Test files (for compute step)
+    CSV.write(joinpath(output_dir, "expert_test.csv"), expert_test)
+    CSV.write(joinpath(output_dir, "lr_data_test.csv"), lr_test)
+
+    # Copy union mapping (needed by model)
+    if isdir(joinpath(base_dir, "data"))
+        mkpath(joinpath(output_dir, "data"))
+        cp(joinpath(base_dir, "data", "union_mapping.csv"),
+           joinpath(output_dir, "data", "union_mapping.csv"), force=true)
+    end
+
+    println()
+    println("Files saved to: $output_dir")
+    println("  text_data.csv  — IDENTICAL to original ($(nrow(text_data)) rows)")
+    println("  expert.csv     — training only ($(nrow(expert_train)) rows)")
+    println("  lr_data.csv    — training only ($(nrow(lr_train)) rows)")
+    println("  expert_test.csv — held-out ($(nrow(expert_test)) rows)")
+    println("  lr_data_test.csv — held-out ($(nrow(lr_test)) rows)")
+
+    return output_dir
+end
+
+# =========================================================================
+# STEP 2: Compute held-out validation metrics
+# =========================================================================
+
+function compute_holdout_metrics(model_file::String, test_dir::String)
+    println()
+    println("="^70)
+    println("COMPUTING HELD-OUT VALIDATION METRICS")
+    println("="^70)
+
+    # Load model output
+    model = CSV.read(model_file, DataFrame)
+    party_col = hasproperty(model, :party_id) ? :party_id : :party
+    println("Model output: $(nrow(model)) party-years")
+
+    # Load test data
+    expert_test = CSV.read(joinpath(test_dir, "expert_test.csv"), DataFrame)
+    lr_test = CSV.read(joinpath(test_dir, "lr_data_test.csv"), DataFrame)
+    println("Held-out expert: $(nrow(expert_test)) observations")
+    println("Held-out LR: $(nrow(lr_test)) observations")
+
+    # Build lookup: (party, year) → model estimates
+    model_lookup = Dict{Tuple{Int,Int}, NamedTuple}()
+    for row in eachrow(model)
+        key = (row[party_col], row.year)
+        model_lookup[key] = (
+            economic_lr = row.economic_lr,
+            galtan = row.galtan,
+            economic_lr_se = hasproperty(row, :economic_lr_se) ? row.economic_lr_se : missing,
+            galtan_se = hasproperty(row, :galtan_se) ? row.galtan_se : missing,
+            economic_lr_q025 = hasproperty(row, :economic_lr_q025) ? row.economic_lr_q025 : missing,
+            economic_lr_q975 = hasproperty(row, :economic_lr_q975) ? row.economic_lr_q975 : missing,
+            galtan_q025 = hasproperty(row, :galtan_q025) ? row.galtan_q025 : missing,
+            galtan_q975 = hasproperty(row, :galtan_q975) ? row.galtan_q975 : missing,
+        )
+    end
+
+    # Map expert variables to dimensions
+    econ_vars = Set(["lrecon_ches", "lrecon_poppa", "lrecon_gps", "lrecon_vparty", "welf_vparty"])
+    galtan_vars = Set(["galtan_ches", "libcon_gps", "immig_vparty", "lgbt_vparty",
+                       "culsup_vparty", "relig_vparty", "gender_vparty"])
+
+    # Process expert test observations
+    results = NamedTuple[]
+
+    for row in eachrow(expert_test)
+        key = (row.party, row.year)
+        haskey(model_lookup, key) || continue
+        m = model_lookup[key]
+
+        if row.var in econ_vars
+            dim = "economic_lr"
+            model_val = m.economic_lr
+            model_q025 = m.economic_lr_q025
+            model_q975 = m.economic_lr_q975
+        elseif row.var in galtan_vars
+            dim = "galtan"
+            model_val = m.galtan
+            model_q025 = m.galtan_q025
+            model_q975 = m.galtan_q975
+        else
+            continue
+        end
+
+        covered = !ismissing(model_q025) && !ismissing(model_q975) &&
+                  row.val >= model_q025 && row.val <= model_q975
+
+        push!(results, (
+            party = row.party,
+            country = row.country,
+            year = row.year,
+            var = row.var,
+            dimension = dim,
+            expert_val = row.val,
+            model_val = model_val,
+            error = row.val - model_val,
+            abs_error = abs(row.val - model_val),
+            covered_95 = ismissing(model_q025) ? missing : covered,
+        ))
+    end
+
+    if isempty(results)
+        println("ERROR: No matching test observations found")
+        return nothing
+    end
+
+    results_df = DataFrame(results)
+
+    # Compute and report metrics
+    println()
+    println("-"^70)
+    println("HELD-OUT VALIDATION RESULTS")
+    println("-"^70)
+
+    # Overall
+    overall_r = cor(results_df.expert_val, results_df.model_val)
+    overall_mae = mean(results_df.abs_error)
+    overall_rmse = sqrt(mean(results_df.error .^ 2))
+    println()
+    println(@sprintf("Overall: r=%.4f, MAE=%.4f, RMSE=%.4f, n=%d",
+                     overall_r, overall_mae, overall_rmse, nrow(results_df)))
+
+    # By dimension
+    println()
+    println("By dimension:")
+    for dim in sort(unique(results_df.dimension))
+        d = filter(r -> r.dimension == dim, results_df)
+        r_val = cor(d.expert_val, d.model_val)
+        mae = mean(d.abs_error)
+        rmse = sqrt(mean(d.error .^ 2))
+        cov = count(skipmissing(d.covered_95)) / count(!ismissing, d.covered_95)
+        println(@sprintf("  %-12s: r=%.4f, MAE=%.4f, RMSE=%.4f, CIC95=%.1f%%, n=%d",
+                         dim, r_val, mae, rmse, 100*cov, nrow(d)))
+    end
+
+    # By source
+    println()
+    println("By source:")
+    for var in sort(unique(results_df.var))
+        v = filter(r -> r.var == var, results_df)
+        nrow(v) < 5 && continue
+        r_val = cor(v.expert_val, v.model_val)
+        mae = mean(v.abs_error)
+        println(@sprintf("  %-20s: r=%.4f, MAE=%.4f, n=%d", var, r_val, mae, nrow(v)))
+    end
+
+    return results_df
+end
+
+# =========================================================================
+# Main
+# =========================================================================
+
+function main()
+    args = ARGS
+
+    if isempty(args) || args[1] == "prepare"
+        output_dir = prepare_holdout_data()
+
+        println()
+        println("="^70)
+        println("NEXT STEPS")
+        println("="^70)
+        println("""
+1. Run model on training data:
+   julia 01_run_model.jl --data-dir $output_dir
+
+2. Run post-estimation on training run output:
+   julia 02_post_estimation.jl
+
+3. Compute held-out metrics:
+   julia scripts/validate_external.jl compute <party_positions_file.csv>
+""")
+
+    elseif args[1] == "compute" && length(args) >= 2
+        model_file = args[2]
+        test_dir = joinpath("validation", "external_split")
+
+        isfile(model_file) || error("Model file not found: $model_file")
+        isdir(test_dir) || error("Test data not found. Run 'prepare' first.")
+
+        results_df = compute_holdout_metrics(model_file, test_dir)
+
+        if results_df !== nothing
+            timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")
+            output_file = joinpath("validation", "external_validation_$(timestamp).csv")
+            CSV.write(output_file, results_df)
+            println()
+            println("Detailed results saved: $output_file")
+        end
+
+    else
+        println("Usage:")
+        println("  julia scripts/validate_external.jl prepare")
+        println("  julia scripts/validate_external.jl compute <party_positions.csv>")
+    end
+end
+
+if abspath(PROGRAM_FILE) == @__FILE__
+    main()
+end

src/julia/validate_uncertainty.jl

@@ -0,0 +1,593 @@
+#!/usr/bin/env julia
+#############################################################################
+## validate_uncertainty.jl
+## Uncertainty validation: Posterior Predictive Coverage (PPC)
+##
+## Following Claassen (2019), this script computes:
+## - PPC: What % of expert values fall within 95% posterior predictive interval?
+## - Also computes 80% PPC for direct Claassen comparison (he reports 60.3%)
+## - Wilson score CI for coverage proportion
+## - Breakdown by survey source and decade
+##
+## Unlike credible interval coverage (which checks θ-CIs), posterior predictive
+## coverage simulates what a new expert observation would look like given the
+## model's beta likelihood, accounting for both position uncertainty AND
+## measurement noise. Well-calibrated models should yield ~95% PPC at 95%.
+##
+## No model re-run needed: reads θ, γ, and φ from existing chain CSV files.
+#############################################################################
+
+using CSV, DataFrames, Statistics, Dates, Printf, Random, JSON
+
+# =============================================================================
+# Utility: Wilson score CI for a proportion
+# =============================================================================
+
+function wilson_ci(p, n; alpha=0.05)
+    if n == 0
+        return (lower=NaN, upper=NaN, se=NaN)
+    end
+    z = 1.96  # For 95% CI
+    denominator = 1 + z^2/n
+    center = (p + z^2/(2n)) / denominator
+    margin = z * sqrt((p*(1-p) + z^2/(4n))/n) / denominator
+    se = sqrt(p * (1-p) / n)
+    return (lower=center - margin, upper=center + margin, se=se)
+end
+
+# =============================================================================
+# STEP 0: Find latest model run
+# =============================================================================
+
+function find_latest_run(base_dir::String="model_outputs")
+    if !isdir(base_dir)
+        error("Model outputs directory not found: $base_dir")
+    end
+    runs = filter(d -> startswith(d, "run_") && isdir(joinpath(base_dir, d)), readdir(base_dir))
+    if isempty(runs)
+        error("No runs found in $base_dir")
+    end
+    sort!(runs, rev=true)
+    latest = joinpath(base_dir, runs[1])
+    println("Using latest run: $latest")
+    return latest
+end
+
+# =============================================================================
+# STEP 1: Load expert_dim.csv from model run data
+# =============================================================================
+
+function load_expert_dim(run_dir::String)
+    expert_dim_file = joinpath(run_dir, "data", "expert_dim.csv")
+    if !isfile(expert_dim_file)
+        error("expert_dim.csv not found in $run_dir/data/")
+    end
+    expert_dim = CSV.read(expert_dim_file, DataFrame)
+    println("Loaded expert_dim.csv: $(nrow(expert_dim)) observations")
+    println("  Unique rr values: $(length(unique(expert_dim.rr_exp_dim)))")
+    println("  Item indices (var_exp_dim): $(sort(unique(expert_dim.var_exp_dim)))")
+    println("  Dimensions (dim_idx_exp): $(sort(unique(expert_dim.dim_idx_exp)))")
+    return expert_dim
+end
+
+# =============================================================================
+# STEP 2: Selectively load chain columns
+# =============================================================================
+
+function load_chains_selective(run_dir::String, needed_rr::Set{Int}, K::Int)
+    """Load only the chain columns we need for posterior predictive checks."""
+    chains_dir = joinpath(run_dir, "chains")
+    chain_files = sort(filter(f -> endswith(f, ".csv") && startswith(f, "chain_"), readdir(chains_dir)))
+
+    if isempty(chain_files)
+        error("No chain files found in $chains_dir")
+    end
+    println("\nLoading $(length(chain_files)) chain files (selective columns)...")
+
+    # Build the set of column names we need
+    needed_cols = Set{String}()
+
+    # theta columns: economic_lr.{rr} and galtan.{rr} for each unique rr
+    for rr in needed_rr
+        push!(needed_cols, "economic_lr.$rr")
+        push!(needed_cols, "galtan.$rr")
+    end
+
+    # Item parameters: gamma_exp_intercept.1-K, gamma_exp_slope.1-K
+    for k in 1:K
+        push!(needed_cols, "gamma_exp_intercept.$k")
+        push!(needed_cols, "gamma_exp_slope.$k")
+    end
+
+    # Precision parameter
+    push!(needed_cols, "phi_exp_dim")
+
+    println("  Need $(length(needed_cols)) columns ($(length(needed_rr)) rr × 2 dims + $(2*K) item params + 1 phi)")
+
+    # Read the header from first chain to identify column indices
+    first_chain_path = joinpath(chains_dir, chain_files[1])
+    header_line = ""
+    open(first_chain_path) do f
+        for line in eachline(f)
+            if !startswith(line, "#")
+                header_line = line
+                break
+            end
+        end
+    end
+
+    all_cols = split(header_line, ",")
+    col_indices = Int[]
+    col_names = String[]
+
+    for (i, col) in enumerate(all_cols)
+        if col in needed_cols
+            push!(col_indices, i)
+            push!(col_names, col)
+        end
+    end
+
+    println("  Found $(length(col_indices))/$(length(needed_cols)) columns in chains")
+
+    if length(col_indices) < length(needed_cols)
+        missing_cols = setdiff(needed_cols, Set(col_names))
+        n_missing = length(missing_cols)
+        sample = collect(missing_cols)[1:min(5, n_missing)]
+        println("  WARNING: Missing columns (showing $( min(5, n_missing))/$n_missing): $sample")
+    end
+
+    # Build a type specification for selective reading
+    # We'll use CSV.read with select parameter
+    select_symbols = Symbol.(col_names)
+
+    all_chains = DataFrame[]
+    for (i, cf) in enumerate(chain_files)
+        path = joinpath(chains_dir, cf)
+        print("  Loading chain $i: $(cf)... ")
+        t = @elapsed begin
+            chain = CSV.read(path, DataFrame; comment="#", select=select_symbols)
+        end
+        println("$(nrow(chain)) samples, $(round(t, digits=1))s")
+        push!(all_chains, chain)
+    end
+
+    combined = vcat(all_chains...)
+    println("Combined: $(nrow(combined)) total posterior draws")
+    return combined
+end
+
+# =============================================================================
+# STEP 3: Compute posterior predictive coverage
+# =============================================================================
+
+function compute_posterior_predictive_cic(chains::DataFrame, expert_dim::DataFrame;
+                                          ci_level::Float64=0.95, seed::Int=42)
+    """
+    Compute posterior predictive coverage for expert dimension observations.
+
+    For each expert observation n with observed value y_n:
+    1. For each posterior draw s:
+       - Get theta_s = theta[dim, rr] (on logit scale, but chains store inv_logit)
+       - Compute mu_s = invlogit(gamma_intercept[k] + gamma_slope[k] * logit(theta_s))
+       - V4 (Beta):          Draw y_pred_s ~ Beta(phi * mu_s, phi * (1 - mu_s))
+       - V5 (Beta-Binomial): Draw y_pred_s ~ Beta(phi * K * mu_s, phi * K * (1 - mu_s))
+         where K = n_experts for that observation
+    2. Compute quantile interval of y_pred draws
+    3. Check if y_n falls within interval
+
+    Returns DataFrame with one row per observation plus coverage indicator.
+    """
+    rng = MersenneTwister(seed)
+
+    alpha_lower = (1 - ci_level) / 2
+    alpha_upper = 1 - alpha_lower
+
+    N = nrow(expert_dim)
+    S = nrow(chains)  # total posterior draws
+
+    # Detect V5 (Beta-Binomial with K-scaling) by presence of n_experts column
+    has_k_scaling = hasproperty(expert_dim, :n_experts)
+    if has_k_scaling
+        k_vec = expert_dim.n_experts
+        println("\nV5 detected: using Beta(phi*K*mu, phi*K*(1-mu)) with per-observation K")
+    else
+        println("\nV4 detected: using Beta(phi*mu, phi*(1-mu))")
+    end
+
+    println("Computing posterior predictive coverage ($(round(Int, 100*ci_level))% level)")
+    println("  Expert observations: $N")
+    println("  Posterior draws: $S")
+
+    # Pre-extract phi vector
+    phi_vec = chains[!, :phi_exp_dim]
+
+    # Pre-extract gamma vectors for each item k
+    K = maximum(expert_dim.var_exp_dim)
+    gamma_int = Dict{Int, Vector{Float64}}()
+    gamma_slope = Dict{Int, Vector{Float64}}()
+    for k in 1:K
+        col_int = Symbol("gamma_exp_intercept.$k")
+        col_slope = Symbol("gamma_exp_slope.$k")
+        if hasproperty(chains, col_int) && hasproperty(chains, col_slope)
+            gamma_int[k] = chains[!, col_int]
+            gamma_slope[k] = chains[!, col_slope]
+        end
+    end
+
+    # Allocate result columns
+    covered = BitVector(undef, N)
+    pred_lower = Vector{Float64}(undef, N)
+    pred_upper = Vector{Float64}(undef, N)
+    pred_median = Vector{Float64}(undef, N)
+
+    # Pre-allocate per-observation draw buffer
+    y_pred = Vector{Float64}(undef, S)
+
+    prog_interval = max(1, N ÷ 20)
+
+    for n in 1:N
+        if n % prog_interval == 0 || n == N
+            pct = round(100 * n / N, digits=1)
+            print("\r  Progress: $pct% ($n / $N)")
+        end
+
+        rr = expert_dim.rr_exp_dim[n]
+        dim = expert_dim.dim_idx_exp[n]
+        k = expert_dim.var_exp_dim[n]
+        y_obs = expert_dim.val[n]
+
+        # Get theta column (chains store inv_logit(theta), i.e. on [0,1] scale)
+        theta_col = dim == 1 ? Symbol("economic_lr.$rr") : Symbol("galtan.$rr")
+
+        if !hasproperty(chains, theta_col) || !haskey(gamma_int, k)
+            # Missing chain data — mark as not covered
+            covered[n] = false
+            pred_lower[n] = NaN
+            pred_upper[n] = NaN
+            pred_median[n] = NaN
+            continue
+        end
+
+        theta_star_vec = chains[!, theta_col]  # inv_logit(theta), i.e. on [0,1]
+        g_int = gamma_int[k]
+        g_slope = gamma_slope[k]
+
+        # Effective concentration: phi for V4, phi * n_experts for V5
+        k_mult = has_k_scaling ? Float64(k_vec[n]) : 1.0
+
+        # For each posterior draw, simulate a predictive observation
+        for s in 1:S
+            theta_star = theta_star_vec[s]
+
+            # Convert back to latent scale for linear predictor
+            # theta_star is inv_logit(theta), so theta = logit(theta_star)
+            # Clamp to avoid Inf
+            theta_star_clamped = clamp(theta_star, 1e-10, 1 - 1e-10)
+            theta_latent = log(theta_star_clamped / (1 - theta_star_clamped))
+
+            # Linear predictor
+            lin = g_int[s] + g_slope[s] * theta_latent
+
+            # Mean of beta
+            mu = 1 / (1 + exp(-lin))
+            mu = clamp(mu, 1e-6, 1 - 1e-6)
+
+            # Beta parameters: phi * K * mu for V5, phi * mu for V4
+            phi = phi_vec[s] * k_mult
+            a = phi * mu
+            b = phi * (1 - mu)
+
+            # Draw from Beta(a, b) via gamma method (no Distributions.jl needed)
+            y_pred[s] = _rand_beta(rng, a, b)
+        end
+
+        # Compute predictive interval
+        sort!(y_pred)
+        idx_lo = max(1, round(Int, alpha_lower * S))
+        idx_hi = min(S, round(Int, alpha_upper * S))
+        idx_med = round(Int, 0.5 * S)
+
+        pred_lower[n] = y_pred[idx_lo]
+        pred_upper[n] = y_pred[idx_hi]
+        pred_median[n] = y_pred[idx_med]
+        covered[n] = (y_obs >= pred_lower[n]) && (y_obs <= pred_upper[n])
+    end
+    println()  # newline after progress
+
+    # Add results to a copy of expert_dim
+    result = DataFrame(
+        rr = expert_dim.rr_exp_dim,
+        dim_idx = expert_dim.dim_idx_exp,
+        var_idx = expert_dim.var_exp_dim,
+        val = expert_dim.val,
+        party = expert_dim.party,
+        country = expert_dim.country,
+        year = expert_dim.year,
+        project = expert_dim.project,
+        var = expert_dim.var,
+        pred_lower = pred_lower,
+        pred_upper = pred_upper,
+        pred_median = pred_median,
+        covered = covered
+    )
+
+    return result
+end
+
+# =============================================================================
+# Beta random variate without Distributions.jl
+# =============================================================================
+
+"""
+    _rand_beta(rng, a, b)
+
+Generate a Beta(a, b) random variate using the Gamma method:
+Beta(a,b) = X/(X+Y) where X ~ Gamma(a), Y ~ Gamma(b).
+Uses Marsaglia & Tsang (2000) for Gamma generation.
+"""
+function _rand_beta(rng::AbstractRNG, a::Float64, b::Float64)
+    x = _rand_gamma(rng, a)
+    y = _rand_gamma(rng, b)
+    return x / (x + y)
+end
+
+"""
+    _rand_gamma(rng, shape)
+
+Generate Gamma(shape, 1) random variate using Marsaglia & Tsang (2000).
+For shape < 1, uses the rejection method with shape+1 then scales.
+"""
+function _rand_gamma(rng::AbstractRNG, shape::Float64)
+    if shape < 1.0
+        # Gamma(a) = Gamma(a+1) * U^(1/a) where U ~ Uniform(0,1)
+        return _rand_gamma(rng, shape + 1.0) * rand(rng)^(1.0 / shape)
+    end
+
+    # Marsaglia & Tsang (2000) for shape >= 1
+    d = shape - 1.0/3.0
+    c = 1.0 / sqrt(9.0 * d)
+
+    while true
+        local x::Float64
+        local v::Float64
+
+        while true
+            x = randn(rng)
+            v = 1.0 + c * x
+            if v > 0.0
+                break
+            end
+        end
+
+        v = v * v * v
+        u = rand(rng)
+
+        if u < 1.0 - 0.0331 * x^2 * x^2
+            return d * v
+        end
+
+        if log(u) < 0.5 * x^2 + d * (1.0 - v + log(v))
+            return d * v
+        end
+    end
+end
+
+# =============================================================================
+# STEP 4: Summarize and save results
+# =============================================================================
+
+function summarize_coverage(result::DataFrame, ci_level::Float64)
+    level_pct = round(Int, 100 * ci_level)
+    println("\n" * "="^60)
+    println("POSTERIOR PREDICTIVE COVERAGE ($level_pct%)")
+    println("="^60)
+
+    # Overall by dimension
+    dim_names = Dict(1 => "economic_lr", 2 => "galtan")
+
+    summary_rows = []
+    for dim in sort(unique(result.dim_idx))
+        subset = filter(r -> r.dim_idx == dim, result)
+        n = nrow(subset)
+        n_covered = sum(subset.covered)
+        ppc = n_covered / n
+        ci = wilson_ci(ppc, n)
+        dim_name = dim_names[dim]
+
+        println(@sprintf("\n  %-15s: %.1f%% [%.1f%%, %.1f%%]  (%d/%d)",
+            dim_name, 100*ppc, 100*ci.lower, 100*ci.upper, n_covered, n))
+
+        push!(summary_rows, (
+            dimension = dim_name,
+            cic = ppc,
+            cic_pct = round(100 * ppc, digits=1),
+            ci_lower = ci.lower,
+            ci_upper = ci.upper,
+            n = n,
+            covered = n_covered
+        ))
+
+        # By project
+        println("\n  By survey source:")
+        by_project = combine(groupby(subset, :project)) do df
+            nc = sum(df.covered)
+            DataFrame(n = nrow(df), covered = nc, cic = nc / nrow(df))
+        end
+        sort!(by_project, :n, rev=true)
+
+        @printf("    %-12s %6s %8s\n", "Project", "N", "PPC")
+        for row in eachrow(by_project)
+            @printf("    %-12s %6d %7.1f%%\n", row.project, row.n, 100*row.cic)
+        end
+
+        # By decade
+        subset_with_decade = copy(subset)
+        subset_with_decade.decade = div.(subset_with_decade.year, 10) .* 10
+        println("\n  By decade:")
+        by_decade = combine(groupby(subset_with_decade, :decade)) do df
+            nc = sum(df.covered)
+            DataFrame(n = nrow(df), covered = nc, cic = nc / nrow(df))
+        end
+        sort!(by_decade, :decade)
+
+        @printf("    %-8s %6s %8s\n", "Decade", "N", "PPC")
+        for row in eachrow(by_decade)
+            @printf("    %-8d %6d %7.1f%%\n", row.decade, row.n, 100*row.cic)
+        end
+    end
+
+    return summary_rows
+end
+
+function save_results(result_95::DataFrame, summary_95, summary_80,
+                      by_project_95::Dict, output_dir::String="validation")
+    if !isdir(output_dir)
+        mkpath(output_dir)
+    end
+
+    timestamp = Dates.format(now(), "yyyy-mm-dd_HH-MM-SS")
+
+    # Summary table (95%)
+    if !isempty(summary_95)
+        summary_df = DataFrame(summary_95)
+        summary_file = joinpath(output_dir, "uncertainty_cic_summary_$timestamp.csv")
+        CSV.write(summary_file, summary_df)
+        println("\nSaved: $summary_file")
+    end
+
+    # Also save 80% summary
+    if !isempty(summary_80)
+        summary80_df = DataFrame(summary_80)
+        summary80_file = joinpath(output_dir, "uncertainty_cic_80pct_summary_$timestamp.csv")
+        CSV.write(summary80_file, summary80_df)
+        println("Saved: $summary80_file")
+    end
+
+    # By-project tables (95%)
+    dim_names = Dict(1 => "economic_lr", 2 => "galtan")
+    for (dim, bp) in by_project_95
+        project_file = joinpath(output_dir, "uncertainty_$(dim_names[dim])_by_project_$timestamp.csv")
+        CSV.write(project_file, bp)
+        println("Saved: $project_file")
+    end
+
+    return summary_95
+end
+
+function print_claassen_comparison(summary_95, summary_80)
+    println("\n" * "="^60)
+    println("COMPARISON WITH CLAASSEN (2019) BENCHMARKS")
+    println("="^60)
+
+    println("\nClaassen's result:")
+    println("  CIC (80% CI): 60.3%")
+    println("  (Using credible intervals for θ, not posterior predictive)")
+    println()
+    println("Our results (posterior predictive):")
+    println()
+
+    println("-"^60)
+    @printf("%-15s %10s %10s %8s\n", "Dimension", "PPC 95%", "PPC 80%", "Status")
+    println("-"^60)
+
+    dim_map_80 = Dict(r.dimension => r for r in summary_80)
+
+    for r in summary_95
+        ppc80 = haskey(dim_map_80, r.dimension) ? dim_map_80[r.dimension].cic : NaN
+        # Well-calibrated: 95% PPC should be near 95%
+        status = r.cic >= 0.90 ? "GOOD" : (r.cic >= 0.80 ? "OK" : "LOW")
+        @printf("%-15s %9.1f%% %9.1f%% %8s\n",
+            r.dimension, 100*r.cic, 100*ppc80, status)
+    end
+    println("-"^60)
+    println()
+    println("Interpretation:")
+    println("  95% PPC ~95% = well-calibrated uncertainty")
+    println("  80% PPC > 60% = exceeds Claassen (2019) benchmark")
+end
+
+# =============================================================================
+# MAIN
+# =============================================================================
+
+function main()
+    println("="^60)
+    println("UNCERTAINTY VALIDATION: Posterior Predictive Coverage")
+    println("="^60)
+    println("Following Claassen (2019) validation framework")
+    println("Posterior predictive intervals account for both position")
+    println("uncertainty AND observation-level measurement noise.")
+    println()
+
+    # Step 0: Parse options and find run directory
+    run_dir = nothing
+    quick_mode = get(ENV, "QUICK_VALIDATION", "0") == "1"
+    for (i, arg) in enumerate(ARGS)
+        if arg == "--run-dir" && i < length(ARGS)
+            run_dir = ARGS[i + 1]
+        elseif startswith(arg, "--run-dir=")
+            run_dir = split(arg, "=", limit=2)[2]
+        elseif arg == "--quick"
+            quick_mode = true
+        end
+    end
+    if run_dir === nothing
+        run_dir = find_latest_run()
+    else
+        println("Using specified run directory: $run_dir")
+    end
+
+    # Step 1: Load expert_dim.csv
+    expert_dim = load_expert_dim(run_dir)
+
+    # Step 2: Selectively load chains
+    needed_rr = Set(expert_dim.rr_exp_dim)
+    K = maximum(expert_dim.var_exp_dim)
+    chains = load_chains_selective(run_dir, needed_rr, K)
+
+    # Step 3a: Compute 95% posterior predictive coverage
+    result_95 = compute_posterior_predictive_cic(chains, expert_dim; ci_level=0.95)
+    summary_95 = summarize_coverage(result_95, 0.95)
+
+    # Step 3b: Compute 80% posterior predictive coverage (Claassen benchmark)
+    # Recompute coverage from the same predictive draws but with 80% quantiles
+    if quick_mode
+        println("\nQUICK MODE: skipping 80% PPC recomputation")
+        summary_80 = [(dimension=r.dimension, n=r.n, covered=r.covered, cic=NaN, ci_level=0.80) for r in summary_95]
+    else
+        println("\n" * "="^60)
+        println("RECOMPUTING WITH 80% LEVEL (Claassen comparison)")
+        println("="^60)
+        result_80 = compute_posterior_predictive_cic(chains, expert_dim; ci_level=0.80, seed=42)
+        summary_80 = summarize_coverage(result_80, 0.80)
+    end
+
+    # Build by-project tables for 95%
+    dim_names = Dict(1 => "economic_lr", 2 => "galtan")
+    by_project_95 = Dict{Int, DataFrame}()
+    for dim in sort(unique(result_95.dim_idx))
+        subset = filter(r -> r.dim_idx == dim, result_95)
+        bp = combine(groupby(subset, :project)) do df
+            nc = sum(df.covered)
+            DataFrame(n = nrow(df), covered = nc, cic = nc / nrow(df))
+        end
+        sort!(bp, :n, rev=true)
+        by_project_95[dim] = bp
+    end
+
+    # Step 4: Save results
+    save_results(result_95, summary_95, summary_80, by_project_95)
+
+    # Step 5: Print Claassen comparison
+    print_claassen_comparison(summary_95, summary_80)
+
+    println("\n" * "="^60)
+    println("VALIDATION COMPLETE")
+    println("="^60)
+
+    return (summary_95=summary_95, summary_80=summary_80)
+end
+
+if abspath(PROGRAM_FILE) == @__FILE__
+    main()
+end

src/r/00_data-management.R

@@ -0,0 +1,311 @@
+# ============================================================
+# 00_data-management.R - Master Data Pipeline Orchestrator
+# ============================================================
+# Coordinates all data processing sub-scripts and produces
+# final output files for the 4D latent trait model.
+#
+# Sub-scripts (run conditionally based on intermediate file existence):
+#   00a_process_manifesto.R  -> manifesto_data.csv
+#   00c_process_poldem.R     -> poldem_data.csv
+#   00d_process_expert.R     -> expert_raw.csv, lr_data_raw.csv
+#   00e_process_morgan.R     -> morgan_data.csv, morgan_lr.csv
+#
+# Final outputs:
+#   text_data.csv - Combined manifesto + PolDem
+#   expert.csv    - Expert survey data (CHES, V-Party, POPPA, GPS)
+#   lr_data.csv   - General left-right anchoring data
+# ============================================================
+
+library(tidyverse)
+library(countrycode)
+
+# Set working directory (works both in RStudio and command line)
+if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
+  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
+}
+
+cat("============================================================\n")
+cat("Data Management Pipeline\n")
+cat("============================================================\n\n")
+
+# ============================================================
+# Configuration: Set to TRUE to force re-run of sub-scripts
+# ============================================================
+
+FORCE_RERUN_MANIFESTO <- FALSE
+FORCE_RERUN_POLDEM <- FALSE
+FORCE_RERUN_EXPERT <- FALSE
+FORCE_RERUN_MORGAN <- FALSE
+
+# ============================================================
+# Step 1: Manifesto Data
+# ============================================================
+
+cat("Step 1: Manifesto data\n")
+if (!file.exists("manifesto_data.csv") || !file.exists("election_data.csv") || FORCE_RERUN_MANIFESTO) {
+  cat("  Running 00a_process_manifesto.R...\n")
+  source("../src/r/00a_process_manifesto.R")
+} else {
+  cat("  Loading cached manifesto_data.csv and election_data.csv...\n")
+}
+manifesto <- read_csv("manifesto_data.csv", show_col_types = FALSE)
+election_data <- read_csv("election_data.csv", show_col_types = FALSE)
+cat(sprintf("  Loaded manifesto: %d rows, %d parties\n", nrow(manifesto), n_distinct(manifesto$party)))
+cat(sprintf("  Loaded election: %d rows, %d parties\n\n", nrow(election_data), n_distinct(election_data$party)))
+
+# ============================================================
+# Step 2: PolDem Media Data
+# ============================================================
+
+cat("Step 2: PolDem media data\n")
+if (!file.exists("poldem_data.csv") || FORCE_RERUN_POLDEM) {
+  cat("  Running 00c_process_poldem.R...\n")
+  source("../src/r/00c_process_poldem.R")
+} else {
+  cat("  Loading cached poldem_data.csv...\n")
+}
+poldem_data <- read_csv("poldem_data.csv", show_col_types = FALSE)
+cat(sprintf("  Loaded: %d rows, %d parties\n\n", nrow(poldem_data), n_distinct(poldem_data$party)))
+
+# ============================================================
+# Step 4: Expert Survey Data
+# ============================================================
+
+cat("Step 3: Expert survey data\n")
+if (!file.exists("expert_raw.csv") || !file.exists("lr_data_raw.csv") || FORCE_RERUN_EXPERT) {
+  cat("  Running 00d_process_expert.R...\n")
+  source("../src/r/00d_process_expert.R")
+} else {
+  cat("  Loading cached expert_raw.csv and lr_data_raw.csv...\n")
+}
+expert_raw <- read_csv("expert_raw.csv", show_col_types = FALSE)
+lr_data_raw <- read_csv("lr_data_raw.csv", show_col_types = FALSE)
+cat(sprintf("  Expert: %d rows, LR: %d rows\n\n", nrow(expert_raw), nrow(lr_data_raw)))
+
+# ============================================================
+# Step 3b: Morgan (1976) Historical Expert Data
+# ============================================================
+
+cat("Step 3b: Morgan (1976) historical L-R data\n")
+
+# First run to generate morgan_data.csv if needed
+if (!file.exists("morgan_data.csv") || FORCE_RERUN_MORGAN) {
+  cat("  Running 00e_process_morgan.R (initial processing)...\n")
+  source("../src/r/00e_process_morgan.R")
+}
+
+# morgan_lr.csv depends on text_data.csv, so we need to check if it needs regeneration
+# It will be generated/regenerated below after text_data is created
+
+# ============================================================
+# Step 4: Combine Text Data Sources
+# ============================================================
+
+cat("Step 4: Combining text data sources\n")
+text_data <- bind_rows(manifesto, poldem_data)
+cat(sprintf("  Combined text_data: %d rows\n", nrow(text_data)))
+
+# Save unfiltered text_data for reproducible mismatch diagnosis
+write_csv(text_data, "text_data_unfiltered.csv")
+cat(sprintf("  Saved unfiltered text_data: %d rows, %d parties\n", nrow(text_data), n_distinct(text_data$party)))
+
+# ============================================================
+# Step 4b: Party Renames (applied before filtering)
+# ============================================================
+# Renames must happen BEFORE the relevance filter so that party IDs
+# match across text_data and expert_raw when computing expert coverage.
+
+# Simple renames only (organizational continuity: same leadership/members)
+simple_renames <- c(
+  `10` = 1816L,    # DE: Greens -> Bündnis90/Grüne
+  `276` = 120L,    # RO: FDSN/PDSR -> PSD (renamed 2001)
+  `8054` = 878L,   # IT: PDS -> DS (renamed 1998)
+  `1696` = 813L,   # IT: MSI -> AN (refounded 1995)
+  `553` = 1968L,   # BE: Vlaams Blok -> Vlaams Belang (refounded 2004)
+  `8058` = 1626L   # IT: Forza Italia (refounded 2013) -> Forza Italia (same party, Berlusconi)
+)
+
+apply_simple_renames <- function(df) {
+  for (old_id in names(simple_renames)) {
+    df <- df %>%
+      mutate(party = ifelse(party == as.integer(old_id), simple_renames[[old_id]], party))
+  }
+  df
+}
+
+cat("\nStep 4b: Party renames\n")
+text_data <- apply_simple_renames(text_data)
+cat(sprintf("  Applied %d renames to text_data\n", length(simple_renames)))
+
+# ============================================================
+# Step 4c: Relevance Filter
+# ============================================================
+# Design: R pipeline filters for RELEVANCE (is this party worth modeling?).
+# Julia pipeline handles INTERPOLATION QUALITY (MAX_GAP=7 segment splitting, MIN_OBS=2).
+# Expert survey coverage is a relevance signal: CHES only covers parties with >1% vote share.
+
+cat("\nStep 4c: Relevance filter\n")
+parties_before <- n_distinct(text_data$party)
+
+# Compute expert coverage per party (with renames applied for consistent matching)
+expert_year_counts <- bind_rows(
+  expert_raw %>% select(party, year),
+  lr_data_raw %>% select(party, year)
+) %>% distinct() %>%
+  apply_simple_renames() %>%
+  distinct() %>%
+  count(party, name = "expert_years")
+
+expert_party_ids <- unique(expert_year_counts$party)
+cat(sprintf("  Parties with expert data: %d\n", length(expert_party_ids)))
+
+# Three-tier relevance filter:
+#   Tier 1: 3+ text data years (always include, regardless of expert data)
+#   Tier 2: 2 text years + any expert data (major newer parties like M5S, ANO, LREM)
+#   Tier 3: 1 text year + 3+ expert survey years (parties with rich expert coverage)
+text_data <- text_data %>%
+  group_by(country, party) %>%
+  mutate(n_years = n_distinct(year)) %>%
+  ungroup() %>%
+  left_join(expert_year_counts, by = "party") %>%
+  mutate(expert_years = replace_na(expert_years, 0L)) %>%
+  mutate(
+    tier = case_when(
+      n_years >= 3 ~ 1L,
+      n_years >= 2 & party %in% expert_party_ids ~ 2L,
+      n_years >= 1 & expert_years >= 3 ~ 3L,
+      TRUE ~ 0L
+    )
+  ) %>%
+  filter(tier > 0) %>%
+  select(-n_years, -expert_years, -tier)
+
+parties_after <- n_distinct(text_data$party)
+cat(sprintf("  Parties before filter: %d\n", parties_before))
+cat(sprintf("  Parties after filter: %d\n", parties_after))
+cat(sprintf("  Parties removed: %d\n\n", parties_before - parties_after))
+
+# ============================================================
+# Step 5: Party Harmonization
+# ============================================================
+
+cat("Step 5: Party harmonization (union-aware)\n")
+
+# Load union mapping to identify constituent parties
+union_map <- read_csv("union_mapping.csv", show_col_types = FALSE)
+
+# Build set of constituent parties whose union is in text_data
+constituent_parties <- union_map %>%
+  filter(manifesto_pf_id %in% unique(text_data$party)) %>%
+  pull(expert_pf_id)
+
+cat(sprintf("  Union mappings loaded: %d rows covering %d unions\n",
+            nrow(union_map), n_distinct(union_map$manifesto_pf_id)))
+cat(sprintf("  Constituent parties with unions in text_data: %d\n",
+            length(unique(constituent_parties))))
+
+# Deduplicate union manifesto rows: where multiple CMP codes map to the same
+# union PF ID with identical content, keep only one set per (party, year, var)
+text_data_before_dedup <- nrow(text_data)
+text_data <- text_data %>%
+  distinct(country, party, year, var, .keep_all = TRUE)
+cat(sprintf("  Text data: %d unique parties after harmonization\n", n_distinct(text_data$party)))
+cat(sprintf("  Text data: deduplicated %d -> %d rows\n", text_data_before_dedup, nrow(text_data)))
+
+# Filter expert data: keep parties in text_data OR constituent parties of unions in text_data
+expert <- expert_raw %>%
+  apply_simple_renames() %>%
+  group_by(country, party, var, year) %>%
+  summarise(
+    val = mean(val, na.rm = TRUE),
+    val_int = first(val_int),
+    n_scale = first(n_scale),
+    n_experts = first(n_experts),
+    project = first(project),
+    type_low = first(type_low),
+    type_high = first(type_high),
+    .groups = "drop"
+  ) %>%
+  filter(party %in% unique(text_data$party) | party %in% constituent_parties)
+
+lr_data <- lr_data_raw %>%
+  apply_simple_renames() %>%
+  group_by(country, party, var, year) %>%
+  summarise(
+    val = mean(val, na.rm = TRUE),
+    val_int = first(val_int),
+    n_scale = first(n_scale),
+    n_experts = first(n_experts),
+    project = first(project),
+    .groups = "drop"
+  ) %>%
+  filter(party %in% unique(text_data$party) | party %in% constituent_parties)
+
+cat(sprintf("  Expert data: %d rows (filtered to text_data parties)\n", nrow(expert)))
+cat(sprintf("  LR data (CHES/POPPA): %d rows (filtered to text_data parties)\n", nrow(lr_data)))
+
+# ============================================================
+# Step 5b: Integrate Morgan L-R Data
+# ============================================================
+
+cat("\nStep 5b: Morgan L-R data integration\n")
+
+# Generate morgan_lr.csv (requires text_data.csv to exist)
+# We need to regenerate it if text_data changed or if forced
+if (!file.exists("morgan_lr.csv") || FORCE_RERUN_MORGAN) {
+  cat("  Generating morgan_lr.csv...\n")
+  # Write text_data first so morgan script can use it
+  write_csv(text_data, "text_data.csv")
+  source("../src/r/00e_process_morgan.R")
+}
+
+# Load and integrate Morgan L-R data
+if (file.exists("morgan_lr.csv")) {
+  morgan_lr <- read_csv("morgan_lr.csv", show_col_types = FALSE) %>%
+    apply_simple_renames() %>%
+    filter(party %in% unique(text_data$party) | party %in% constituent_parties)
+
+  cat(sprintf("  Morgan L-R: %d rows (filtered to text_data parties)\n", nrow(morgan_lr)))
+  cat(sprintf("  Morgan parties: %d\n", n_distinct(morgan_lr$party)))
+  cat(sprintf("  Morgan year range: %d-%d\n", min(morgan_lr$year), max(morgan_lr$year)))
+
+  # Combine with existing lr_data
+  lr_data_before <- nrow(lr_data)
+  lr_data <- bind_rows(lr_data, morgan_lr) %>%
+    arrange(country, party, year, var)
+
+  cat(sprintf("  Combined LR data: %d rows (+%d from Morgan)\n",
+              nrow(lr_data), nrow(lr_data) - lr_data_before))
+} else {
+  cat("  Warning: morgan_lr.csv not found, skipping Morgan integration\n")
+}
+
+cat("\n")
+
+# ============================================================
+# Step 6: Write Final Outputs
+# ============================================================
+
+cat("Step 6: Writing final outputs\n")
+
+write_csv(text_data, "text_data.csv")
+write_csv(expert, "expert.csv")
+write_csv(lr_data, "lr_data.csv")
+
+cat("\n============================================================\n")
+cat("Pipeline Complete!\n")
+cat("============================================================\n\n")
+
+cat("Output files written:\n")
+cat(sprintf("  text_data.csv: %d rows\n", nrow(text_data)))
+cat(sprintf("    - Manifesto: %d rows\n", sum(grepl("_manifesto", text_data$var))))
+cat(sprintf("    - PolDem:    %d rows\n", sum(grepl("_poldem", text_data$var))))
+cat(sprintf("  expert.csv: %d rows\n", nrow(expert)))
+cat(sprintf("  lr_data.csv: %d rows\n", nrow(lr_data)))
+cat(sprintf("    - CHES:   %d rows\n", sum(lr_data$var == "lr_ches")))
+cat(sprintf("    - POPPA:  %d rows\n", sum(lr_data$var == "lr_poppa")))
+cat(sprintf("    - Morgan: %d rows\n", sum(lr_data$var == "lr_morgan")))
+
+cat("\nUnique parties in text_data:", n_distinct(text_data$party), "\n")
+cat("Countries:", paste(sort(unique(text_data$country)), collapse = ", "), "\n")
+cat("Year range:", min(text_data$year, na.rm = TRUE), "-", max(text_data$year, na.rm = TRUE), "\n")

src/r/00a_process_manifesto.R

@@ -0,0 +1,178 @@
+# ============================================================
+# 00a_process_manifesto.R - Manifesto Project Data Processing
+# ============================================================
+# Processes Manifesto Project data for the 4D latent trait model
+# Input: $PARTY2D_RAW_DATA_DIR/manifesto/MPDataset_MPDS2025a.csv
+# Output: manifesto_data.csv
+# ============================================================
+
+library(tidyverse)
+library(countrycode)
+library(purrr)
+
+# Set working directory (works both in RStudio and command line)
+if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
+  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
+}
+
+cat("Processing Manifesto Project data...\n")
+
+raw_data_dir <- Sys.getenv(
+  "PARTY2D_RAW_DATA_DIR",
+  unset = file.path("..", "..", "_local", "raw")
+)
+manifesto_raw_path <- file.path(raw_data_dir, "manifesto", "MPDataset_MPDS2025a.csv")
+
+# ============================================================
+# PartyFacts Linkage
+# ============================================================
+
+partyfacts_raw <- read_csv('partyfacts-external-parties.csv', show_col_types = FALSE)
+manifesto_link <- partyfacts_raw %>%
+  filter(dataset_key == "manifesto") %>%
+  transmute(id = dataset_party_id,
+            country = countrycode(country, origin = 'iso3c', destination = "iso2c"),
+            party = partyfacts_id,
+            party = ifelse(party == 622, 604, party))
+
+# ============================================================
+# Load Manifesto Data
+# ============================================================
+
+manifesto_data <- read_csv(manifesto_raw_path, show_col_types = FALSE)
+
+# ============================================================
+# CMP Code Mapping to 4 Dimensions
+# ============================================================
+
+vars <- tribble(
+  ~type,          ~subtype,                 ~per_var,  ~stance,    ~label,
+  # pro_market
+  "pro_market",   "Market Regulation",      "per401",  "Positive", "Free Market Economy",
+  "pro_market",   "Economic Liberalization","per402",  "Positive", "Incentives: Positive",
+  "pro_market",   "Market Regulation",      "per407",  "Positive", "Protectionism: Negative",
+  "pro_market",   "Economic Liberalization","per414",  "Positive", "Economic Orthodoxy",
+  "pro_market",   "Economic Liberalization","per505",  "Positive", "Welfare State Limitation",
+  "pro_market",   "Economic Liberalization","per507",  "Positive", "Education Limitation",
+  "pro_market",   "Economic Liberalization","per702",  "Positive", "Labour Groups: Negative",
+  "pro_market",   "Market Regulation",      "per406",  "Negative", "Protectionism: Positive",
+  "pro_market",   "Market Regulation",      "per412",  "Negative", "Controlled Economy",
+  "pro_market",   "Economic Liberalization","per504",  "Negative", "Welfare State Expansion",
+  # pro_welfare
+  "pro_welfare",  "Economic Intervention",  "per403",  "Positive", "Market Regulation",
+  "pro_welfare",  "Economic Intervention",  "per404",  "Positive", "Economic Planning",
+  "pro_welfare",  "Economic Intervention",  "per412",  "Positive", "Controlled Economy",
+  "pro_welfare",  "Economic Intervention",  "per413",  "Positive", "Nationalisation",
+  "pro_welfare",  "Social Services",        "per504",  "Positive", "Welfare State Expansion",
+  "pro_welfare",  "Social Services",        "per506",  "Positive", "Education Expansion",
+  "pro_welfare",  "Economic Intervention",  "per701",  "Positive", "Labour Groups: Positive",
+  "pro_welfare",  "Economic Intervention",  "per401",  "Negative", "Free Market Economy",
+  "pro_welfare",  "Social Services",        "per505",  "Negative", "Welfare State Limitation",
+  # cosmopolitan
+  "cosmopolitan", "Internationalism",       "per107",  "Positive", "Internationalism: Positive",
+  "cosmopolitan", "Internationalism",       "per108",  "Positive", "European Community/Union: Positive",
+  "cosmopolitan", "Multiculturalism",       "per607",  "Positive", "Multiculturalism: Positive",
+  "cosmopolitan", "Multiculturalism",       "per201",  "Positive", "Freedom and Human Rights",
+  "cosmopolitan", "Multiculturalism",       "per604",  "Positive", "traditional Morality: Negative",
+  "cosmopolitan", "Internationalism",       "per109",  "Negative", "Internationalism: Negative",
+  "cosmopolitan", "Multiculturalism",       "per601",  "Negative", "National Way of Life: Positive",
+  # traditional
+  "traditional",  "National Identity",      "per109",  "Positive", "Internationalism: Negative",
+  "traditional",  "Conservative Morality",  "per110",  "Positive", "European Community/Union: Negative",
+  "traditional",  "National Identity",      "per601",  "Positive", "National Way of Life: Positive",
+  "traditional",  "Conservative Morality",  "per603",  "Positive", "traditional Morality: Positive",
+  "traditional",  "Conservative Morality",  "per608",  "Positive", "Multiculturalism: Negative",
+  "traditional",  "Conservative Morality",  "per605",  "Positive", "Law and Order: Positive",
+  "traditional",  "National Identity",      "per107",  "Negative", "Internationalism: Positive",
+  "traditional",  "Conservative Morality",  "per607",  "Negative", "Multiculturalism: Positive"
+)
+
+# ============================================================
+# Process Manifesto Data
+# ============================================================
+
+manifesto <- vars %>%
+  pmap_dfr(~ manifesto_data %>%
+             transmute(country = countrycode(countryname, origin = 'country.name', destination = 'iso2c'),
+                       year = as.numeric(format(as.Date(edate, format = "%d/%m/%Y"), "%Y")),
+                       id = as.character(party),
+                       count = round(.data[[..3]]),
+                       var = ..3,
+                       label = ..5,
+                       type = ..1,
+                       subtype = ..2,
+                       stance = ..4,
+                       project = 'Manifesto Project') %>%
+             left_join(manifesto_link, by = c("id", "country")) %>%
+             select(-id)) %>%
+  group_by(party, country, year, subtype) %>%
+  summarise(
+    positive = sum(count[stance == "Positive"], na.rm = TRUE),
+    sample = sum(count, na.rm = TRUE),
+    type = first(type),
+    project = first(project),
+    .groups = "drop"
+  ) %>%
+  na.omit() %>%
+  rename(var = subtype) %>%
+  # Convert to bipolar bridge structure (type_high/type_low)
+  mutate(
+    type_high = case_when(
+      type == "pro_welfare" ~ "pro_welfare",
+      type == "pro_market" ~ "pro_market",
+      type == "cosmopolitan" ~ "cosmopolitan",
+      type == "traditional" ~ "traditional"
+    ),
+    type_low = case_when(
+      type %in% c("pro_welfare", "pro_market") ~ ifelse(type == "pro_welfare", "pro_market", "pro_welfare"),
+      type %in% c("cosmopolitan", "traditional") ~ ifelse(type == "cosmopolitan", "traditional", "cosmopolitan")
+    )
+  ) %>%
+  select(-type) %>%
+  # Add _manifesto suffix to variable names
+  mutate(var = paste0(tolower(gsub(" ", "_", var)), "_manifesto"))
+
+# ============================================================
+# NOTE: Temporal continuity filter moved to 00_data-management.R
+# This allows exempting parties that appear in parliamentary data
+# (parties in parliament are by definition not fringe parties)
+# ============================================================
+
+cat("Skipping temporal filter (applied in 00_data-management.R after combining with parl data)\n")
+cat(sprintf("  Parties: %d\n", n_distinct(manifesto$party)))
+
+# ============================================================
+# Write Output
+# ============================================================
+
+write_csv(manifesto, "manifesto_data.csv")
+cat(sprintf("Output: manifesto_data.csv (%d rows, %d parties)\n",
+            nrow(manifesto), n_distinct(manifesto$party)))
+
+# ============================================================
+# Election Data Extraction (vote shares)
+# ============================================================
+
+cat("\nExtracting election data (pervote)...\n")
+
+election_data <- manifesto_data %>%
+  transmute(
+    country = countrycode(countryname, origin = 'country.name', destination = 'iso2c'),
+    year = as.numeric(format(as.Date(edate, format = "%d/%m/%Y"), "%Y")),
+    id = as.character(party),
+    pervote = pervote
+  ) %>%
+  left_join(manifesto_link, by = c("id", "country")) %>%
+  select(-id) %>%
+  filter(!is.na(party), !is.na(pervote)) %>%
+  # Keep one row per (party, country, year) — take max pervote if duplicates
+  group_by(party, country, year) %>%
+  summarise(pervote = max(pervote, na.rm = TRUE), .groups = "drop") %>%
+  arrange(country, party, year)
+
+write_csv(election_data, "election_data.csv")
+cat(sprintf("Output: election_data.csv (%d rows, %d parties)\n",
+            nrow(election_data), n_distinct(election_data$party)))
+
+# Export manifesto_link for use by other scripts
+# (poldem also needs it for CMP linkage)

src/r/00c_process_poldem.R

@@ -0,0 +1,161 @@
+# ============================================================
+# 00c_process_poldem.R - PolDem Media Data Processing
+# ============================================================
+# Processes PolDem (Political Deliberation in the Media) data
+# for the 4D latent trait model
+#
+# Input: $PARTY2D_RAW_DATA_DIR/poldem/poldem-election_all.csv (sentence-level)
+# Output: poldem_data.csv (party-year-var aggregates)
+# ============================================================
+
+library(tidyverse)
+library(countrycode)
+
+# Set working directory (works both in RStudio and command line)
+if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
+  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
+}
+
+cat("Processing PolDem media data...\n")
+
+raw_data_dir <- Sys.getenv(
+  "PARTY2D_RAW_DATA_DIR",
+  unset = file.path("..", "..", "_local", "raw")
+)
+poldem_raw_path <- file.path(raw_data_dir, "poldem", "poldem-election_all.csv")
+
+# ============================================================
+# PartyFacts Linkage (via CMP party IDs)
+# ============================================================
+
+partyfacts_raw <- read_csv('partyfacts-external-parties.csv', show_col_types = FALSE)
+manifesto_link <- partyfacts_raw %>%
+  filter(dataset_key == "manifesto") %>%
+  transmute(cmp = as.numeric(dataset_party_id),  # Convert to numeric for join
+            country_pf = countrycode(country, origin = 'iso3c', destination = "iso2c"),
+            party = partyfacts_id,
+            party = ifelse(party == 622, 604, party))
+
+# ============================================================
+# Issue Category Mapping to 4 Dimensions
+# ============================================================
+# For positive direction: type_high is the active trait
+# For negative direction: we flip (same data, just contributes to the opposite trait)
+
+poldem_mapping <- tribble(
+  ~issue_cat,     ~dimension,  ~type_high,      ~type_low,
+  # Economic dimension
+  "ecolib",       "economic",  "pro_market",    "pro_welfare",   # Economic liberalization
+  "welfare",      "economic",  "pro_welfare",   "pro_market",    # Welfare state
+  # Final exclusion: the PolDem economic-reform category is intentionally
+  # omitted because item-response diagnostics showed that it did not load
+  # substantively onto the economic latent trait.
+  # Cultural dimension
+  "immig",        "cultural",  "cosmopolitan",  "traditional",   # Immigration (pro = cosmopolitan)
+  "cultlib",      "cultural",  "cosmopolitan",  "traditional",   # Cultural liberalism
+  "nationalism",  "cultural",  "traditional",   "cosmopolitan",  # Nationalism (pro = traditional)
+  "europe",       "cultural",  "cosmopolitan",  "traditional",   # EU integration (pro = cosmopolitan)
+  "euro",         "cultural",  "cosmopolitan",  "traditional",   # Euro currency (pro = cosmopolitan)
+  "defense",      "cultural",  "traditional",   "cosmopolitan",  # Defense (pro = traditional)
+  "security",     "cultural",  "traditional",   "cosmopolitan"   # Security/law-order (pro = traditional)
+)
+
+cat(sprintf("  Using %d issue categories\n", nrow(poldem_mapping)))
+
+# ============================================================
+# Load and Clean PolDem Data
+# ============================================================
+
+poldem_raw <- read_csv(poldem_raw_path, show_col_types = FALSE)
+cat(sprintf("  Raw PolDem data: %d rows\n", nrow(poldem_raw)))
+
+poldem <- poldem_raw %>%
+  # Fix country codes
+  mutate(country = case_when(
+    iso2code == "AU" ~ "AT",  # Austria (PolDem uses AU instead of AT)
+    iso2code == "UK" ~ "GB",  # United Kingdom
+    TRUE ~ iso2code
+  )) %>%
+  # Extract year from article date (format: YYYY-MM-DD)
+  mutate(year = suppressWarnings(as.numeric(substr(date_art, 1, 4)))) %>%
+  # Filter to valid issue categories only
+  filter(issue_cat %in% poldem_mapping$issue_cat) %>%
+  # Convert direction to numeric and filter out neutral/NA
+  mutate(direction = as.numeric(direction)) %>%
+  filter(!is.na(direction) & direction != 0) %>%
+  # Remove rows with invalid years
+  filter(!is.na(year))
+
+cat(sprintf("  After filtering: %d rows (valid issues, non-neutral)\n", nrow(poldem)))
+
+# ============================================================
+# Link to PartyFacts via CMP codes
+# ============================================================
+
+poldem <- poldem %>%
+  mutate(cmp = as.numeric(cmp)) %>%
+  left_join(manifesto_link, by = "cmp") %>%
+  filter(!is.na(party))
+
+# Report linkage
+n_linked <- nrow(poldem)
+n_unlinked <- nrow(poldem_raw %>%
+                     filter(issue_cat %in% poldem_mapping$issue_cat) %>%
+                     mutate(direction = as.numeric(direction)) %>%
+                     filter(!is.na(direction) & direction != 0)) - n_linked
+
+cat(sprintf("  Linked to PartyFacts: %d rows\n", n_linked))
+if (n_unlinked > 0) {
+  cat(sprintf("  Warning: %d rows could not be linked (missing CMP mapping)\n", n_unlinked))
+}
+
+# ============================================================
+# Aggregate to Party-Year-Issue Level
+# Using round(sum()) for weak direction values (0.5, -0.5)
+# ============================================================
+
+poldem_agg <- poldem %>%
+  left_join(poldem_mapping, by = "issue_cat") %>%
+  group_by(party, country, year, issue_cat, type_high, type_low) %>%
+  summarise(
+    # Sum positive directions (0.5 and 1), then round
+    positive = round(sum(direction[direction > 0])),
+    # Sum absolute directions for sample (all non-neutral), then round
+    sample = round(sum(abs(direction))),
+    n_obs = n(),
+    .groups = "drop"
+  ) %>%
+  # Minimum 3 observations per group
+  filter(n_obs >= 3) %>%
+  select(-n_obs)
+
+cat(sprintf("  After aggregation: %d party-year-issue observations\n", nrow(poldem_agg)))
+
+# ============================================================
+# Format Output (matching manifesto structure)
+# ============================================================
+
+poldem_data <- poldem_agg %>%
+  mutate(
+    var = paste0(issue_cat, "_poldem"),
+    project = "PolDem"
+  ) %>%
+  select(party, country, year, var, positive, sample, type_high, type_low, project)
+
+# ============================================================
+# Write Output
+# ============================================================
+
+write_csv(poldem_data, "poldem_data.csv")
+
+cat(sprintf("\nOutput: poldem_data.csv\n"))
+cat(sprintf("  Total rows: %d\n", nrow(poldem_data)))
+cat(sprintf("  Unique parties: %d\n", n_distinct(poldem_data$party)))
+cat(sprintf("  Countries: %s\n", paste(sort(unique(poldem_data$country)), collapse = ", ")))
+cat(sprintf("  Year range: %d-%d\n", min(poldem_data$year, na.rm = TRUE), max(poldem_data$year, na.rm = TRUE)))
+cat("\n  Rows by issue category:\n")
+poldem_data %>%
+  group_by(var) %>%
+  summarise(n = n(), .groups = "drop") %>%
+  arrange(desc(n)) %>%
+  print()

src/r/00d_process_expert.R

@@ -0,0 +1,580 @@
+# ============================================================
+# 00d_process_expert.R - Expert Survey Data Processing
+# ============================================================
+# Processes expert survey data from multiple sources:
+# - Chapel Hill Expert Survey (CHES)
+# - V-Party Dataset
+# - POPPA
+# - GPS (Norris)
+#
+# Outputs: expert_raw.csv, lr_data_raw.csv
+#
+# V5 changes:
+# - val_int (integer rounded to nearest scale point) and n_scale columns
+# - n_experts column preserved (not dropped)
+# - V-Party cultural expansion: 5 native items replace GPS ep_v6_lib_cons
+# - V-Party economic expansion: v2pawelf added
+# - Reverse-coding for V-Party cultural + welfare items
+# ============================================================
+
+library(tidyverse)
+library(countrycode)
+library(haven)
+library(foreign)
+
+# Set working directory (works both in RStudio and command line)
+if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
+  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
+}
+
+cat("Processing expert survey data...\n")
+
+# ============================================================
+# PartyFacts Linkage for CHES
+# ============================================================
+
+partyfacts_raw <- read_csv('partyfacts-external-parties.csv', show_col_types = FALSE)
+ches_link <- partyfacts_raw %>%
+  filter(dataset_key == "ches") %>%
+  transmute(id = dataset_party_id,
+            country = countrycode(country, origin = 'iso3c', destination = "iso2c"),
+            party = partyfacts_id)
+
+# ============================================================
+# Expert Count Tables (from individual response files)
+# ============================================================
+
+cat("  Loading expert count tables from individual response files...\n")
+
+# CHES 2024: dual lookup (party_id primary, country+name fallback for ID mismatches)
+ches24_exp_raw <- read_csv('~/data/ches/CHES_2024_expert_level.csv', show_col_types = FALSE)
+ches24_exp_by_id <- ches24_exp_raw %>%
+  group_by(party_id) %>%
+  summarise(n_experts_id = as.integer(n_distinct(id)), .groups = "drop")
+ches24_exp_by_name <- ches24_exp_raw %>%
+  mutate(country_iso2 = countrycode(cname, origin = "country.name", destination = "iso2c")) %>%
+  group_by(country_iso2, party_name) %>%
+  summarise(n_experts_name = as.integer(n_distinct(id)), .groups = "drop")
+
+ches_ca_expert_counts <- read_csv('~/data/ches/CHES_CA2023_expert_level.csv', show_col_types = FALSE) %>%
+  group_by(party_id) %>%
+  summarise(n_experts = as.integer(n_distinct(expert)), .groups = "drop")
+
+ches_la_expert_counts <- read_csv('~/data/ches/CHES_LA2020_expert_level.csv', show_col_types = FALSE) %>%
+  group_by(party_id) %>%
+  summarise(n_experts = as.integer(n_distinct(expert_id)), .groups = "drop")
+
+ches_il_expert_counts <- read_csv('~/data/ches/CHES_IL_expert_level.csv', show_col_types = FALSE) %>%
+  group_by(party_id, year) %>%
+  summarise(n_experts = as.integer(n_distinct(id)), .groups = "drop")
+
+# ============================================================
+# Chapel Hill Expert Survey (CHES) - 1999-2019
+# ============================================================
+
+cat("  Processing CHES 1999-2019...\n")
+
+ches <- read_csv('~/data/ches/1999-2019_CHES_dataset_means(v3).csv', show_col_types = FALSE) %>%
+  rename(country_id = country) %>%
+  left_join(readRDS('~/data/ches/link.rds'), by = "country_id") %>%
+  transmute(country = countrycode(country, origin = 'country.name', destination = 'iso2c'),
+            vote = vote,
+            year = year,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = as.integer(expert),
+            lrecon_ches = lrecon/10,
+            galtan_ches = galtan/10) %>%
+  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
+  mutate(n_scale = 10L) %>%
+  left_join(ches_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id) %>%
+  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
+         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
+
+# ============================================================
+# CHES 2024 Update
+# ============================================================
+
+cat("  Processing CHES 2024...\n")
+
+# Country code lookup for CHES 2024 format
+country_lookup <- c(
+  "be" = "BE", "dk" = "DK", "ge" = "DE", "gr" = "GR", "esp" = "ES",
+  "fr" = "FR", "irl" = "IE", "it" = "IT", "nl" = "NL", "uk" = "GB",
+  "por" = "PT", "aus" = "AT", "fin" = "FI", "sv" = "SE", "bul" = "BG",
+  "cz" = "CZ", "est" = "EE", "hun" = "HU", "lat" = "LV", "lith" = "LT",
+  "pol" = "PL", "rom" = "RO", "slo" = "SK", "sle" = "SI", "cro" = "HR",
+  "tur" = "TR", "nor" = "NO", "swi" = "CH", "mal" = "MT", "cyp" = "CY",
+  "ice" = "IS"
+)
+
+convert_country_codes <- function(codes) {
+  result <- country_lookup[codes]
+  result[is.na(result)] <- codes[is.na(result)]
+  return(unname(result))
+}
+
+ches24 <- read_csv('~/data/ches/CHES_2024_final_v2.csv', show_col_types = FALSE) %>%
+  mutate(country_iso2 = convert_country_codes(country)) %>%
+  left_join(ches24_exp_by_id, by = "party_id") %>%
+  left_join(ches24_exp_by_name, by = c("country_iso2", "party" = "party_name")) %>%
+  transmute(country = country_iso2,
+            vote = vote,
+            year = 2024,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = coalesce(n_experts_id, n_experts_name),
+            lrecon_ches = lrecon/10,
+            galtan_ches = galtan/10) %>%
+  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
+  mutate(n_scale = 10L) %>%
+  left_join(ches_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id) %>%
+  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
+         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
+
+ches <- bind_rows(ches, ches24)
+
+# ============================================================
+# CHES Canada 2023
+# ============================================================
+
+cat("  Processing CHES Canada 2023...\n")
+
+ches_ca <- read_csv('~/data/ches/CHES_CA2023.csv', show_col_types = FALSE) %>%
+  filter(!is.na(partyfacts_id)) %>%
+  left_join(ches_ca_expert_counts, by = "party_id") %>%
+  transmute(country = "CA",
+            year = 2023,
+            party = partyfacts_id,
+            project = 'CHES',
+            n_experts = n_experts,
+            lrecon_ches = lrecon/10,
+            galtan_ches = galtan/10) %>%
+  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
+  mutate(n_scale = 10L) %>%
+  filter(!is.na(val), !is.na(party)) %>%
+  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
+         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
+
+ches <- bind_rows(ches, ches_ca)
+cat(sprintf("    CHES Canada: %d observations\n", nrow(ches_ca)))
+
+# ============================================================
+# CHES Latin America 2020
+# ============================================================
+
+cat("  Processing CHES Latin America 2020...\n")
+
+ches_la_link <- read_csv('~/data/ches/ches_la_link.csv', show_col_types = FALSE) %>%
+  transmute(id = as.character(ches_party_id),
+            party = partyfacts_id)
+
+ches_la <- read_csv('~/data/ches/ches_la_2020_aggregate_level_v01.csv', show_col_types = FALSE) %>%
+  left_join(ches_la_expert_counts, by = "party_id") %>%
+  transmute(country = country_abb,
+            year = 2020,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = n_experts,
+            lrecon_ches = lrecon/10,
+            galtan_ches = galtan/10) %>%
+  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
+  mutate(n_scale = 10L) %>%
+  left_join(ches_la_link, by = "id") %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id) %>%
+  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
+         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
+
+ches <- bind_rows(ches, ches_la)
+cat(sprintf("    CHES Latin America: %d observations\n", nrow(ches_la)))
+
+# ============================================================
+# CHES Israel 2021-2022
+# ============================================================
+
+cat("  Processing CHES Israel 2021-2022...\n")
+
+ches_il_link <- read_csv('~/data/ches/ches_israel_link.csv', show_col_types = FALSE) %>%
+  transmute(id = as.character(ches_party_id),
+            party = partyfacts_id)
+
+ches_il <- read_csv('~/data/ches/CHES_ISRAEL_means_2021_2022.csv', show_col_types = FALSE) %>%
+  left_join(ches_il_expert_counts, by = c("party_id", "year")) %>%
+  transmute(country = "IL",
+            year = year,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = n_experts,
+            lrecon_ches = lrecon/10,
+            galtan_ches = galtan/10) %>%
+  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
+  mutate(n_scale = 10L) %>%
+  left_join(ches_il_link, by = "id") %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id) %>%
+  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
+         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
+
+ches <- bind_rows(ches, ches_il)
+cat(sprintf("    CHES Israel: %d observations\n", nrow(ches_il)))
+
+cat(sprintf("    CHES total: %d observations\n", nrow(ches)))
+
+# ============================================================
+# CHES General Left-Right (for anchoring)
+# ============================================================
+
+cat("  Processing CHES LR anchoring data...\n")
+
+ches_lr <- read_csv('~/data/ches/1999-2019_CHES_dataset_means(v3).csv', show_col_types = FALSE) %>%
+  rename(country_id = country) %>%
+  left_join(readRDS('~/data/ches/link.rds'), by = "country_id") %>%
+  transmute(country = countrycode(country, origin = 'country.name', destination = 'iso2c'),
+            vote = vote,
+            year = year,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = as.integer(expert),
+            val = lrgen/10,
+            var = 'lr_ches',
+            n_scale = 10L) %>%
+  left_join(ches_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id)
+
+ches24_lr <- read_csv('~/data/ches/CHES_2024_final_v2.csv', show_col_types = FALSE) %>%
+  mutate(country_iso2 = convert_country_codes(country)) %>%
+  left_join(ches24_exp_by_id, by = "party_id") %>%
+  left_join(ches24_exp_by_name, by = c("country_iso2", "party" = "party_name")) %>%
+  transmute(country = country_iso2,
+            vote = vote,
+            year = 2024,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = coalesce(n_experts_id, n_experts_name),
+            val = lrgen/10,
+            var = 'lr_ches',
+            n_scale = 10L) %>%
+  left_join(ches_link, by = c("id", "country")) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id)
+
+# CHES Canada LR
+ches_ca_lr <- read_csv('~/data/ches/CHES_CA2023.csv', show_col_types = FALSE) %>%
+  filter(!is.na(partyfacts_id)) %>%
+  left_join(ches_ca_expert_counts, by = "party_id") %>%
+  transmute(country = "CA",
+            year = 2023,
+            party = partyfacts_id,
+            project = 'CHES',
+            n_experts = n_experts,
+            val = lrgen/10,
+            var = 'lr_ches',
+            n_scale = 10L) %>%
+  filter(!is.na(val), !is.na(party))
+
+# CHES Latin America LR
+ches_la_lr <- read_csv('~/data/ches/ches_la_2020_aggregate_level_v01.csv', show_col_types = FALSE) %>%
+  left_join(ches_la_expert_counts, by = "party_id") %>%
+  transmute(country = country_abb,
+            year = 2020,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = n_experts,
+            val = lrgen/10,
+            var = 'lr_ches',
+            n_scale = 10L) %>%
+  left_join(ches_la_link, by = "id") %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id)
+
+# CHES Israel LR
+ches_il_lr <- read_csv('~/data/ches/CHES_ISRAEL_means_2021_2022.csv', show_col_types = FALSE) %>%
+  left_join(ches_il_expert_counts, by = c("party_id", "year")) %>%
+  transmute(country = "IL",
+            year = year,
+            id = as.character(party_id),
+            project = 'CHES',
+            n_experts = n_experts,
+            val = lrgen/10,
+            var = 'lr_ches',
+            n_scale = 10L) %>%
+  left_join(ches_il_link, by = "id") %>%
+  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
+  select(-id)
+
+ches_lr <- bind_rows(ches_lr, ches24_lr, ches_ca_lr, ches_la_lr, ches_il_lr)
+
+# ============================================================
+# V-Party Dataset (V5: expanded to 7 variables)
+# ============================================================
+
+cat("  Processing V-Party...\n")
+
+vparty_raw <- readRDS('~/data/v-party/V-Dem-CPD-Party-V2.rds')
+
+# Economic 1: v2pariglef_osp (0-6 scale, higher = more right, NO reverse)
+vparty_econ1 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2pariglef_nr),
+    val = v2pariglef_osp / 6,
+    val_int = as.integer(round(v2pariglef_osp)),
+    n_scale = 6L,
+    var = "lrecon_vparty",
+    type_low = "pro_welfare",
+    type_high = "pro_market"
+  ) %>%
+  na.omit()
+
+# Economic 2 (NEW): v2pawelf_osp (0-5 scale, higher = more welfare = LEFT, REVERSE)
+vparty_econ2 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2pawelf_nr),
+    val = 1 - v2pawelf_osp / 5,
+    val_int = 5L - as.integer(round(v2pawelf_osp)),
+    n_scale = 5L,
+    var = "welf_vparty",
+    type_low = "pro_welfare",
+    type_high = "pro_market"
+  ) %>%
+  na.omit()
+
+# Cultural 1 (NEW): v2paimmig_osp (0-4 scale, higher = more pro-immigration = GAL, REVERSE)
+vparty_cult1 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2paimmig_nr),
+    val = 1 - v2paimmig_osp / 4,
+    val_int = 4L - as.integer(round(v2paimmig_osp)),
+    n_scale = 4L,
+    var = "immig_vparty",
+    type_low = "cosmopolitan",
+    type_high = "traditional"
+  ) %>%
+  na.omit()
+
+# Cultural 2 (NEW): v2palgbt_osp (0-4 scale, higher = more pro-LGBT = GAL, REVERSE)
+vparty_cult2 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2palgbt_nr),
+    val = 1 - v2palgbt_osp / 4,
+    val_int = 4L - as.integer(round(v2palgbt_osp)),
+    n_scale = 4L,
+    var = "lgbt_vparty",
+    type_low = "cosmopolitan",
+    type_high = "traditional"
+  ) %>%
+  na.omit()
+
+# Cultural 3 (NEW): v2paculsup_osp (0-4 scale, higher = less cultural superiority = GAL, REVERSE)
+vparty_cult3 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2paculsup_nr),
+    val = 1 - v2paculsup_osp / 4,
+    val_int = 4L - as.integer(round(v2paculsup_osp)),
+    n_scale = 4L,
+    var = "culsup_vparty",
+    type_low = "cosmopolitan",
+    type_high = "traditional"
+  ) %>%
+  na.omit()
+
+# Cultural 4 (NEW): v2parelig_osp (0-4 scale, higher = less religious = GAL, REVERSE)
+vparty_cult4 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2parelig_nr),
+    val = 1 - v2parelig_osp / 4,
+    val_int = 4L - as.integer(round(v2parelig_osp)),
+    n_scale = 4L,
+    var = "relig_vparty",
+    type_low = "cosmopolitan",
+    type_high = "traditional"
+  ) %>%
+  na.omit()
+
+# Cultural 5 (NEW): v2pagender_osp (0-4 scale, higher = more pro-gender equality = GAL, REVERSE)
+vparty_cult5 <- vparty_raw %>%
+  transmute(
+    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
+    year = year,
+    party = pf_party_id,
+    project = "V-Party",
+    n_experts = as.integer(v2pagender_nr),
+    val = 1 - v2pagender_osp / 4,
+    val_int = 4L - as.integer(round(v2pagender_osp)),
+    n_scale = 4L,
+    var = "gender_vparty",
+    type_low = "cosmopolitan",
+    type_high = "traditional"
+  ) %>%
+  na.omit()
+
+vparty <- bind_rows(vparty_econ1, vparty_econ2,
+                    vparty_cult1, vparty_cult2, vparty_cult3,
+                    vparty_cult4, vparty_cult5)
+
+cat(sprintf("    V-Party: %d observations (7 variables)\n", nrow(vparty)))
+cat(sprintf("      lrecon: %d, welf: %d\n", nrow(vparty_econ1), nrow(vparty_econ2)))
+cat(sprintf("      immig: %d, lgbt: %d, culsup: %d, relig: %d, gender: %d\n",
+            nrow(vparty_cult1), nrow(vparty_cult2), nrow(vparty_cult3),
+            nrow(vparty_cult4), nrow(vparty_cult5)))
+
+# ============================================================
+# POPPA Dataset
+# ============================================================
+
+cat("  Processing POPPA...\n")
+
+poppa <- readRDS('~/data/POPPA/poppa_integrated_v2.rds') %>%
+  transmute(country = countrycode(country_short, origin = "iso3c", destination = "iso2c"),
+            party = partyfacts_id,
+            val = lrecon/10,
+            var = "lrecon_poppa",
+            type_low = "pro_welfare",
+            type_high = "pro_market",
+            n_experts = as.integer(n_experts),
+            n_scale = 10L,
+            year = as.numeric(sub(".*-\\s*(\\d+)", "\\1", wave)),
+            project = "POPPA") %>%
+  na.omit()
+
+cat(sprintf("    POPPA: %d observations\n", nrow(poppa)))
+
+# POPPA General LR
+poppa_lr <- readRDS('~/data/POPPA/poppa_integrated_v2.rds') %>%
+  transmute(country = countrycode(country_short, origin = "iso3c", destination = "iso2c"),
+            party = partyfacts_id,
+            val = lroverall/10,
+            var = "lr_poppa",
+            n_experts = as.integer(n_experts),
+            n_scale = 10L,
+            year = as.numeric(sub(".*-\\s*(\\d+)", "\\1", wave)),
+            project = "POPPA") %>%
+  na.omit()
+
+# ============================================================
+# GPS (Norris) Survey
+# ============================================================
+
+cat("  Processing GPS...\n")
+
+gps <- read.delim("~/data/GPS_norris/Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab") %>%
+  transmute(n_experts = as.integer(Experts),
+            lrecon_gps = as.numeric(V4_Scale)/10,
+            libcon_gps = as.numeric(V6_Scale)/10,
+            party = ID_PartyFacts,
+            country = countrycode(ifelse(ISO == "MAC", "MKD", ISO), origin = "iso3c", destination = "iso2c"),
+            year = 2019,
+            n_scale = 10L,
+            project = "GPS") %>%
+  pivot_longer(cols = lrecon_gps:libcon_gps, names_to = 'var', values_to = 'val') %>%
+  mutate(type_low = ifelse(var == "lrecon_gps", "pro_welfare", "cosmopolitan"),
+         type_high = ifelse(var == "lrecon_gps", "pro_market", "traditional")) %>%
+  na.omit()
+
+cat(sprintf("    GPS: %d observations\n", nrow(gps)))
+
+# ============================================================
+# Combine Expert Data
+# ============================================================
+
+cat("  Combining expert surveys...\n")
+
+expert_raw <- select(ches, -vote) %>%
+  bind_rows(vparty) %>%
+  bind_rows(gps) %>%
+  bind_rows(poppa) %>%
+  unique() %>%
+  arrange(country, party, year, var) %>%
+  filter(!is.na(val), !is.na(party), !is.na(country), !is.na(var))
+
+# Compute val_int for datasets that don't have it pre-computed
+# V-Party already has val_int; CHES/GPS/POPPA need it computed from val * n_scale
+expert_raw <- expert_raw %>%
+  mutate(
+    val_int = ifelse(is.na(val_int), as.integer(round(val * n_scale)), val_int),
+    val_int = pmin(pmax(val_int, 0L), n_scale)
+  )
+
+# Boundary adjustments for continuous val (avoid exact 0 or 1 for Stan prior means)
+expert_raw <- expert_raw %>%
+  mutate(
+    val = case_when(
+      val == 0 ~ val + 1e-4,
+      val == 1 ~ val - 1e-4,
+      TRUE ~ val
+    ))
+
+# ============================================================
+# Combine LR Data
+# ============================================================
+
+lr_data_raw <- ches_lr %>%
+  bind_rows(poppa_lr) %>%
+  select(-any_of("vote"))
+
+# Boundary adjustments for continuous val (avoid exact 0 or 1)
+lr_data_raw <- lr_data_raw %>%
+  mutate(
+    val = case_when(
+      val == 0 ~ val + 1e-4,
+      val == 1 ~ val - 1e-4,
+      TRUE ~ val
+    ))
+
+# Compute val_int for LR data
+lr_data_raw <- lr_data_raw %>%
+  mutate(
+    val_int = as.integer(round(val * n_scale)),
+    val_int = pmin(pmax(val_int, 0L), n_scale)
+  )
+
+# ============================================================
+# Write Outputs
+# ============================================================
+
+write_csv(expert_raw, "expert_raw.csv")
+write_csv(lr_data_raw, "lr_data_raw.csv")
+
+cat(sprintf("\nOutputs written:\n"))
+cat(sprintf("  expert_raw.csv: %d rows\n", nrow(expert_raw)))
+cat(sprintf("  lr_data_raw.csv: %d rows\n", nrow(lr_data_raw)))
+
+cat("\n  Expert data by source:\n")
+expert_raw %>%
+  group_by(project) %>%
+  summarise(n = n(), .groups = "drop") %>%
+  print()
+
+cat("\n  New columns check:\n")
+cat(sprintf("  val_int range: %d - %d\n", min(expert_raw$val_int), max(expert_raw$val_int)))
+cat(sprintf("  n_scale values: %s\n", paste(sort(unique(expert_raw$n_scale)), collapse = ", ")))
+cat(sprintf("  n_experts non-NA: %d / %d\n", sum(!is.na(expert_raw$n_experts)), nrow(expert_raw)))

src/r/00e_process_morgan.R

@@ -0,0 +1,388 @@
+# 00e_process_morgan.R
+# Process Morgan (1976) expert party position data
+#
+# Source: Morgan, Michael-John (1976). "The Modelling of Governmental
+# Coalition Formation: A Policy-Based Approach with Interval Measurement."
+# PhD dissertation, University of Michigan.
+#
+# Data extracted from Appendix B.3 (Tables B.3.1-B.3.12) via OCR.
+# Position scores are 25%-truncated means (midmeans) from expert surveys.
+# Scale: 0-100 (left-right)
+
+library(tidyverse)
+
+cat("Processing Morgan (1976) expert party position data...\n")
+
+# Load raw extracted data
+morgan_raw <- read_csv("morgan_positions_raw.csv", show_col_types = FALSE)
+
+cat(sprintf("Loaded %d party-period observations from %d countries\n",
+            nrow(morgan_raw), n_distinct(morgan_raw$country)))
+
+# Load PartyFacts linkage data
+partyfacts <- read_csv("partyfacts-external-parties.csv", show_col_types = FALSE)
+
+# Filter to Morgan dataset entries
+morgan_pf <- partyfacts %>%
+  filter(dataset_key == "morgan") %>%
+  select(country, name_short, name_english, year_first, year_last,
+         external_id, partyfacts_id) %>%
+  rename(party_abbrev_pf = name_short)
+
+cat(sprintf("Found %d Morgan parties in PartyFacts\n", nrow(morgan_pf)))
+
+# Map extracted abbreviations to PartyFacts abbreviations
+# Some adjustments needed due to OCR/transcription differences
+abbrev_map <- tribble(
+  ~country, ~party_abbrev, ~party_abbrev_pf,
+  # Denmark
+  "DNK", "SOCd", "SOCD",
+  "DNK", "SOCL", "SOCL",
+  "DNK", "COMM", "COMM",
+  "DNK", "RAD", "RAD",
+  "DNK", "LIB", "LIB",
+  "DNK", "CONS", "CONS",
+  "DNK", "LS", "LS",
+  "DNK", "LC", "LC",
+  "DNK", "JUST", "JUST",
+  # Finland
+  "FIN", "SKDL", "SKDL",
+  "FIN", "SOCd", "SOCD",
+  "FIN", "PROG", "PROG",
+  "FIN", "AGR", "AGR",
+  "FIN", "SWPP", "SWPP",
+  "FIN", "CONS", "CONS",
+  "FIN", "NPF", "NPF",
+  "FIN", "PDEM", "PDEM",
+  "FIN", "SDWS", "SDWS",
+  "FIN", "CENT", "CENT",
+  "FIN", "FRP", "FRP",
+  "FIN", "LIB", "LIB",
+  # Iceland
+  "ISL", "COMM", "COMM",
+  "ISL", "SOCd", "SOCD",
+  "ISL", "PROG", "PROG",
+  "ISL", "LIB", "LIB",
+  "ISL", "INDP", "INDP",
+  "ISL", "CONS", "CONS",
+  "ISL", "LLIB", "LLIB",
+  # Norway
+  "NOR", "LAB", "LAB",
+  "NOR", "LIB", "LIB",
+  "NOR", "AGR", "AGR",
+  "NOR", "CONS", "CONS",
+  "NOR", "COMM", "COMM",
+  "NOR", "SOCL", "SOCL",
+  "NOR", "CHPP", "CHPP",
+  "NOR", "CENT", "CENT",
+  # Sweden
+  "SWE", "COMM", "COMM",
+  "SWE", "SOCd", "SOCD",
+  "SWE", "AGR", "AGR",
+  "SWE", "LIB", "LIB",
+  "SWE", "CONS", "CONS",
+  "SWE", "CENT", "CENT",
+  # Netherlands
+  "NLD", "CPN", "CPN",
+  "NLD", "SOCd", "SOCD",
+  "NLD", "RAD", "RAD",
+  "NLD", "KVP", "KVP",
+  "NLD", "CHU", "CHU",
+  "NLD", "LIB", "LIB",
+  "NLD", "ARP", "ARP",
+  "NLD", "SGP", "SGP",
+  "NLD", "NSB", "NSB",
+  "NLD", "PVDA", "PVDA",
+  "NLD", "VVD", "VVD",
+  "NLD", "PSP", "PSP",
+  "NLD", "PPR", "PPR",
+  "NLD", "D66", "D66",
+  "NLD", "DS70", "DS70",
+  "NLD", "GPV", "GPV",
+  "NLD", "BP", "BP",
+  # Belgium
+  "BEL", "COMM", "COMM",
+  "BEL", "POB", "POB",
+  "BEL", "CATH", "CATH",
+  "BEL", "LIB", "LIB",
+  "BEL", "FNAT", "FNAT",
+  "BEL", "REX", "REX",
+  "BEL", "PSB", "PSB",
+  "BEL", "RW", "RW",
+  "BEL", "PSC", "PSC",
+  "BEL", "FDF", "FDF",
+  "BEL", "VOLK", "VOLK",
+  "BEL", "PLP", "PLP",
+  # France (Fourth Republic)
+  "FRA", "PCF", "PCF",
+  "FRA", "SFIO", "SFIO",
+  "FRA", "MRP", "MRP",
+  "FRA", "RDA", "RDA",
+  "FRA", "UDSR", "UDSR",
+  "FRA", "RAD", "RAD",
+  "FRA", "RS", "RS",
+  "FRA", "RPF", "RPF",
+  "FRA", "AR", "AR",
+  "FRA", "ARS", "ARS",
+  "FRA", "RI", "RI",
+  "FRA", "CNIP", "CNIP",
+  "FRA", "PUS", "PUS",
+  "FRA", "PAYS", "PAYS",
+  "FRA", "AP", "AP",
+  "FRA", "PRL", "PRL",
+  "FRA", "POUJ", "POUJ",
+  # Weimar Germany
+  "DEU", "KPD", "KPD",
+  "DEU", "SDAP", "SDAP",
+  "DEU", "DDP", "DDP",
+  "DEU", "DZP", "DZP",
+  "DEU", "BVP", "BVP",
+  "DEU", "DVP", "DVP",
+  "DEU", "RDMW", "RDMW",
+  "DEU", "LVP", "LVP",
+  "DEU", "DNVP", "DNVP",
+  "DEU", "NAZI", "NAZI",
+  # Italy
+  "ITA", "PCI", "PCI",
+  "ITA", "PSIU", "PSIU",
+  "ITA", "PSI", "PSI",
+  "ITA", "PSDI", "PSDI",
+  "ITA", "PRI", "PRI",
+  "ITA", "DC", "DC",
+  "ITA", "PLI", "PLI",
+  "ITA", "MON", "MON",
+  "ITA", "MSI", "MSI",
+  # Luxembourg
+  "LUX", "COMM", "COMM",
+  "LUX", "SOCd", "SOCD",
+  "LUX", "CSOC", "CSOC",
+  "LUX", "GRPD", "GRPD",
+  # Israel
+  "ISR", "RAKA", "RAKA",
+  "ISR", "MAKI", "MAKI",
+  "ISR", "MAPM", "MAPM",
+  "ISR", "MADT", "MADT",
+  "ISR", "ADUT", "ADUT",
+  "ISR", "MAAR", "MAAR",
+  "ISR", "LAB", "LAB",
+  "ISR", "MAPI", "MAPI",
+  "ISR", "PAUG", "PAUG",
+  "ISR", "RAFI", "RAFI",
+  "ISR", "PROG", "PROG",
+  "ISR", "ILIB", "ILIB",
+  "ISR", "NRP", "NRP",
+  "ISR", "URF", "URF",
+  "ISR", "LIB", "LIB",
+  "ISR", "NATL", "NATL",
+  "ISR", "TORA", "TORA",
+  "ISR", "LIKD", "LIKD",
+  "ISR", "ZION", "ZION",
+  "ISR", "GHAL", "GHAL",
+  "ISR", "AGDT", "AGDT",
+  "ISR", "HRUT", "HRUT"
+)
+
+# Some parties in raw data that don't have exact matches - need special handling
+# (e.g., parties that only exist in one period in PartyFacts but appear in both)
+# We'll join using the period-based matching
+
+# Expand periods to years for matching
+morgan_expanded <- morgan_raw %>%
+  mutate(
+    year_start = as.integer(str_extract(period, "^\\d{4}")),
+    year_end = as.integer(str_extract(period, "\\d{4}$"))
+  )
+
+# Join with abbreviation map
+morgan_mapped <- morgan_expanded %>%
+  left_join(abbrev_map, by = c("country", "party_abbrev"))
+
+# Check for unmatched abbreviations
+unmatched_abbrev <- morgan_mapped %>%
+  filter(is.na(party_abbrev_pf)) %>%
+  distinct(country, party_abbrev)
+
+if (nrow(unmatched_abbrev) > 0) {
+  cat("\nWarning: Unmatched abbreviations:\n")
+  print(unmatched_abbrev)
+}
+
+# Join with PartyFacts
+morgan_joined <- morgan_mapped %>%
+  left_join(morgan_pf, by = c("country", "party_abbrev_pf")) %>%
+  # For parties with overlapping periods, use period overlap
+  mutate(
+    period_overlap = pmax(0,
+      pmin(year_end, year_last) - pmax(year_start, year_first) + 1)
+  ) %>%
+  # Keep best match per party-period (max overlap)
+  group_by(country, party_abbrev, period) %>%
+  slice_max(period_overlap, n = 1, with_ties = FALSE) %>%
+  ungroup()
+
+# Check for unmatched parties
+unmatched <- morgan_joined %>%
+  filter(is.na(partyfacts_id)) %>%
+  distinct(country, party_abbrev, party_name, period)
+
+if (nrow(unmatched) > 0) {
+  cat(sprintf("\n%d party-periods without PartyFacts match:\n", nrow(unmatched)))
+  print(unmatched)
+}
+
+# Dedup: when multiple abbreviations map to the same PF ID, keep only one
+matched <- morgan_joined %>%
+  filter(!is.na(partyfacts_id)) %>%
+  group_by(country, partyfacts_id, period) %>%
+  slice(1) %>%
+  ungroup()
+
+cat(sprintf("\nMatched %d of %d party-period observations (%.1f%%)\n",
+            nrow(matched), nrow(morgan_raw),
+            100 * nrow(matched) / nrow(morgan_raw)))
+
+# Normalize position to [0,1] scale
+# Original scale: 0-100
+# Apply boundary adjustments like other expert data
+eps <- 0.005
+morgan_processed <- matched %>%
+  mutate(
+    # Normalize to [0,1]
+    lr_morgan = position / 100,
+    # Apply boundary adjustments
+    lr_morgan = case_when(
+      lr_morgan <= 0 ~ eps,
+      lr_morgan >= 1 ~ 1 - eps,
+      TRUE ~ lr_morgan
+    ),
+    # Calculate standard error (sd / sqrt(n))
+    lr_morgan_se = (sd / 100) / sqrt(n_surveys),
+    # Set minimum SE for extreme parties (sd=0)
+    lr_morgan_se = pmax(lr_morgan_se, 0.01)
+  ) %>%
+  select(
+    country,
+    partyfacts_id,
+    period,
+    year_start,
+    year_end,
+    party_abbrev,
+    party_name,
+    lr_morgan,
+    lr_morgan_se,
+    n_surveys
+  ) %>%
+  arrange(country, year_start, lr_morgan)
+
+# Summary statistics
+cat("\nSummary of processed Morgan data:\n")
+cat(sprintf("  Countries: %d\n", n_distinct(morgan_processed$country)))
+cat(sprintf("  Parties: %d\n", n_distinct(morgan_processed$partyfacts_id)))
+cat(sprintf("  Observations: %d\n", nrow(morgan_processed)))
+
+# Distribution of positions
+cat("\nPosition distribution:\n")
+print(summary(morgan_processed$lr_morgan))
+
+# Write output
+write_csv(morgan_processed, "morgan_data.csv")
+cat(sprintf("\nWrote morgan_data.csv with %d rows\n", nrow(morgan_processed)))
+
+# Also provide a summary by country and period
+summary_by_country <- morgan_processed %>%
+  group_by(country, period) %>%
+  summarise(
+    n_parties = n(),
+    mean_pos = mean(lr_morgan),
+    sd_pos = sd(lr_morgan),
+    .groups = "drop"
+  )
+
+cat("\nSummary by country and period:\n")
+print(summary_by_country, n = 50)
+
+# ============================================================
+# Generate lr_data-compatible output for pipeline integration
+# ============================================================
+
+cat("\n============================================================\n")
+cat("Generating lr_data-compatible output (postwar only)\n")
+cat("============================================================\n")
+
+# Load text_data to get party-years with manifesto/PolDem coverage
+text_data <- read_csv("text_data.csv", show_col_types = FALSE)
+
+# Convert Morgan ISO3 country codes to ISO2 (matching text_data format)
+iso3_to_iso2 <- c(
+  "DNK" = "DK", "FIN" = "FI", "ISL" = "IS", "NOR" = "NO", "SWE" = "SE",
+  "NLD" = "NL", "BEL" = "BE", "DEU" = "DE", "FRA" = "FR", "ITA" = "IT",
+  "LUX" = "LU", "ISR" = "IL"
+)
+
+# Filter to postwar periods only (1945+)
+morgan_postwar <- morgan_processed %>%
+  filter(year_end >= 1945) %>%
+  mutate(country_iso2 = iso3_to_iso2[country])
+
+cat(sprintf("Postwar Morgan observations: %d party-periods\n", nrow(morgan_postwar)))
+cat(sprintf("Countries: %s\n", paste(unique(morgan_postwar$country_iso2), collapse = ", ")))
+
+# Get unique party-years from text_data
+text_party_years <- text_data %>%
+  select(party, country, year) %>%
+  distinct()
+
+cat(sprintf("Unique party-years in text_data: %d\n", nrow(text_party_years)))
+
+# For each Morgan party-period, expand to all years where that party has text data
+# within the Morgan period range (1945-1973 for postwar)
+morgan_lr <- morgan_postwar %>%
+  # Join with text_data party-years
+  # Many-to-many is expected: one Morgan party-period maps to multiple years
+  inner_join(
+    text_party_years,
+    by = c("partyfacts_id" = "party", "country_iso2" = "country"),
+    relationship = "many-to-many"
+  ) %>%
+  # Keep only years within the Morgan period
+  filter(year >= year_start & year <= year_end) %>%
+  # Format for lr_data.csv compatibility
+  transmute(
+    country = country_iso2,
+    party = partyfacts_id,
+    var = "lr_morgan",
+    year = year,
+    val = lr_morgan,
+    project = "Morgan",
+    # Morgan's continuous 0-100 scale is discretized to 10 points (matching CHES
+    # resolution) with the actual number of experts. The reconstructed sum
+    # round(mean × K × 10) is analogous to how CHES means are handled.
+    # See docs/k_scaling_validation.md Section 4.
+    n_scale = 10L,
+    val_int = as.integer(round(lr_morgan * 10)),
+    n_experts = as.integer(n_surveys)
+  ) %>%
+  distinct() %>%
+  arrange(country, party, year)
+
+cat(sprintf("\nGenerated %d lr_morgan observations\n", nrow(morgan_lr)))
+cat(sprintf("  Unique parties: %d\n", n_distinct(morgan_lr$party)))
+cat(sprintf("  Year range: %d-%d\n", min(morgan_lr$year), max(morgan_lr$year)))
+
+# Summary by country
+morgan_lr_summary <- morgan_lr %>%
+  group_by(country) %>%
+  summarise(
+    n_parties = n_distinct(party),
+    n_obs = n(),
+    year_min = min(year),
+    year_max = max(year),
+    .groups = "drop"
+  )
+
+cat("\nMorgan L-R data by country:\n")
+print(morgan_lr_summary, n = 20)
+
+# Write morgan_lr.csv
+write_csv(morgan_lr, "morgan_lr.csv")
+cat(sprintf("\nWrote morgan_lr.csv with %d rows\n", nrow(morgan_lr)))

src/sh/check_raw_data.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+project_root="$(cd "$repo_root/.." && pwd -P)"
+raw_data_dir="${PARTY2D_RAW_DATA_DIR:-$project_root/_local/raw}"
+
+required_files=(
+  "poldem/poldem-election_all.csv"
+)
+
+optional_files=(
+  "manifesto/MPDataset_MPDS2025a.csv"
+)
+
+echo "Raw data directory: $raw_data_dir"
+echo
+echo "Required raw inputs for regeneration:"
+
+missing=0
+for rel in "${required_files[@]}"; do
+  path="$raw_data_dir/$rel"
+  if [ -s "$path" ]; then
+    bytes="$(wc -c < "$path")"
+    read -r sha _ < <(sha256sum "$path")
+    echo "  OK       $rel ($bytes bytes, sha256=$sha)"
+  else
+    echo "  MISSING  $rel"
+    missing=1
+  fi
+done
+
+echo
+echo "Optional raw inputs used only if cached processed files are regenerated:"
+for rel in "${optional_files[@]}"; do
+  path="$raw_data_dir/$rel"
+  if [ -s "$path" ]; then
+    bytes="$(wc -c < "$path")"
+    read -r sha _ < <(sha256sum "$path")
+    echo "  OK       $rel ($bytes bytes, sha256=$sha)"
+  else
+    echo "  MISSING  $rel"
+  fi
+done
+
+if [ "$missing" -ne 0 ]; then
+  echo
+  echo "At least one required raw input is missing." >&2
+  echo "See docs/RAW_DATA_SOURCES.md for download/local-placement instructions." >&2
+  exit 1
+fi
+
+echo
+echo "Required raw data preflight passed."

src/sh/show_run_progress.sh

@@ -0,0 +1,74 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+repo_root="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd -P)"
+logs_dir="$repo_root/outputs/logs"
+model_dir="$repo_root/outputs/model_outputs/latest"
+
+latest_log="$(ls "$logs_dir"/full_run_*.log 2>/dev/null | sort | tail -n 1 || true)"
+latest_run="$(ls -d "$model_dir"/run_* 2>/dev/null | sort | tail -n 1 || true)"
+
+echo "party2d full-run progress"
+echo "repo: $repo_root"
+echo
+
+if [[ -n "$latest_log" ]]; then
+  echo "Latest durable log: $latest_log"
+  echo "--- last 40 log lines ---"
+  tail -n 40 "$latest_log"
+else
+  echo "No durable full_run_*.log found under $logs_dir"
+fi
+
+echo
+if [[ -n "$latest_run" ]]; then
+  echo "Latest model run: $latest_run"
+  metrics="$latest_run/diagnostics/run_metrics.json"
+  if [[ -f "$metrics" ]]; then
+    echo "Metrics: $metrics"
+    echo "--- key metrics ---"
+    py="$(command -v python3 || command -v python || true)"
+    if [[ -z "$py" && -x /run/current-system/sw/bin/python3 ]]; then
+      py=/run/current-system/sw/bin/python3
+    fi
+    if [[ -z "$py" ]]; then
+      echo "No Python interpreter found for JSON summary; metrics file is available at: $metrics"
+      exit 0
+    fi
+    "$py" - "$metrics" <<'PY'
+import json, sys
+path = sys.argv[1]
+with open(path) as f:
+    m = json.load(f)
+print("status:", m.get("status"))
+cfg = m.get("config", {})
+print("chains:", cfg.get("num_chains"), "warmup:", cfg.get("num_warmup"), "samples:", cfg.get("num_samples"), "max_depth:", cfg.get("max_depth"), "refresh:", cfg.get("refresh"))
+timing = m.get("timing", {})
+if timing:
+    print("walltime_minutes:", timing.get("walltime_minutes"))
+    print("reported_warmup_seconds:", timing.get("reported_warmup_seconds"))
+    print("reported_sampling_seconds:", timing.get("reported_sampling_seconds"))
+agg = m.get("aggregate", {})
+if agg:
+    print("divergences:", agg.get("divergences"), "treedepth_hits:", agg.get("treedepth_hits"), "mean_leapfrog:", agg.get("mean_leapfrog"))
+print("cmdstan_config_verified:", m.get("cmdstan_config_verified"))
+PY
+  else
+    echo "No archived metrics yet. If the run is still sampling, watch the durable log with:"
+    if [[ -n "$latest_log" ]]; then
+      echo "  tail -f \"$latest_log\""
+    else
+      echo "  tail -f outputs/logs/full_run_<timestamp>.log"
+    fi
+  fi
+else
+  echo "No completed model run found under $model_dir"
+fi
+
+echo
+echo "To follow a running job live:"
+if [[ -n "$latest_log" ]]; then
+  echo "  tail -f \"$latest_log\""
+else
+  echo "  tail -f outputs/logs/full_run_<timestamp>.log"
+fi