Clean public release repository

2026-06-15 18:12:13 +02:00
parent 8c920739ea
commit 6c1e40fffe
30 changed files with 221 additions and 2691 deletions
@@ -9,7 +9,6 @@ Code and processed model inputs for generating two-dimensional party-position es
 - `models/` — Stan model specification.
 - `data/` — processed party-level inputs used by the Julia/Stan model.
 - `metadata/` — data dictionary and source-support documentation.
 - `docs/` — raw data source documentation, coding decisions, and operational notes.
 - `diagnostics/` — repository diagnostics report regenerated after model estimation.
 - `data/releases/` — release-ready data files, checksums, and diagnostics report.
@@ -36,7 +35,7 @@ Two inputs require user-provided access/material:
 - **Manifesto Project**: users must obtain the source data through their own Manifesto Project access.
 - **Morgan historical expert data**: `morgan_positions_raw.csv` is not publicly downloadable; it can be provided on request and should be placed locally under `_local/raw/morgan/`.
-The setup workflow never overwrites committed files in `data/`. See `data-setup/README.md` and `docs/RAW_DATA_SOURCES.md` for exact commands and source details.
+The setup workflow never overwrites committed files in `data/`. See `data-setup/README.md` for exact commands and source details.
 Run the full source-data setup workflow with:
@@ -2,7 +2,7 @@
 # 02_build_model_inputs.R - Master Data Pipeline Orchestrator
 # ============================================================
 # Coordinates all data processing sub-scripts and produces
-# final output files for the 4D latent trait model. By default this writes only
+# final output files for the two-dimensional party-position model. By default this writes only
 # to local-only directories under _local/ and never overwrites committed data/.
 #
 # Sub-scripts (run conditionally based on intermediate file existence):
@@ -1,7 +1,7 @@
 # ============================================================
 # process_manifesto.R - Manifesto Project Data Processing
 # ============================================================
-# Processes Manifesto Project data for the 4D latent trait model
+# Processes Manifesto Project data for the two-dimensional party-position model
 # Input: $PARTY2D_RAW_DATA_DIR/manifesto/MPDataset_MPDS2025a.csv
 # Output: manifesto_data.csv
 # ============================================================
@@ -2,7 +2,7 @@
 # process_poldem.R - PolDem Media Data Processing
 # ============================================================
 # Processes PolDem (Political Deliberation in the Media) data
-# for the 4D latent trait model
+# for the two-dimensional party-position model
 #
 # Input: $PARTY2D_RAW_DATA_DIR/poldem/poldem-election_all.csv (sentence-level)
 # Output: poldem_data.csv (party-year-var aggregates)
@@ -8,6 +8,6 @@ This directory contains only the processed, model-ready inputs used by the Julia
 - `union_mapping.csv`
 - `party_families.csv`
-Original raw source files and intermediate build products are not stored here. To regenerate the processed inputs, place raw files in a local directory and set `PARTY2D_RAW_DATA_DIR`; see `../data-setup/README.md` and `../docs/RAW_DATA_SOURCES.md`.
+Original raw source files and intermediate build products are not stored here. To regenerate the processed inputs, place raw files in a local directory and set `PARTY2D_RAW_DATA_DIR`; see `../data-setup/README.md`.
 Generated outputs and temporary staging files are ignored by git.
@@ -1,3 +1,3 @@
 6b9aeda20489000983e459e4eb19d2788ed035eaab5deb9a5e010338670b2ff1  party_2d_election_year_panel_v0.zip
 8f3973011fb57818656199d020d00d9a666c9414b085a3d66feb1124b629caab  party_2d_annual_model_output_v0.zip
-c92a006bb931d1eb2df6e48d0fb71278c2ce92cde20b499165f37bee03a3081b  party_2d_diagnostics_report_v0.pdf
+b99f7ef0e8a4c821183a2a0f957752303479e78e5afb685f173fd17f60039b3e  party_2d_diagnostics_report_v0.pdf
@@ -1,107 +1,107 @@
-manifesto_pf_id,manifesto_name,expert_pf_id,expert_name,country,relationship,status,detection_method
+manifesto_pf_id,manifesto_name,expert_pf_id,expert_name,country,status
-3889,PJ,6648,PF-PJ,AR,Peronist faction no independent manifesto,implemented,manual
+3889,PJ,6648,PF-PJ,AR,implemented
-6161,FAP,1365,PS,AR,"LLM: PS (Socialist Party) was a core constituent of FAP (Frente Amplio Progresista), which published joint manifestos.",implemented,llm_verified
+6161,FAP,1365,PS,AR,implemented
-6161,FAP,6160,FR,AR,"LLM: FR (Frente Renovador) joined FAP in some elections, but also ran independently; likely constituent in FAP manifestos.",implemented,llm_verified
+6161,FAP,6160,FR,AR,implemented
-6161,FAP,6554,FPCyS,AR,"LLM: FPCyS (Frente Progresista, Cívico y Social) included PS and others; manifestos often under FAP or similar coalitions.",implemented,llm_verified
+6161,FAP,6554,FPCyS,AR,implemented
-486,LPA,1998,LP,AU,"LLM: The Liberal Party (LP, PF ID: 1998) is the predecessor and constituent of the Liberal Party of Australia (LPA, PF ID: 486); manifestos are published under LPA, not LP.",implemented,llm_verified
+486,LPA,1998,LP,AU,implemented
-1743,NPA,338,NAT,AU,"LLM: The National Party (NAT) is the predecessor and constituent of the National Party of Australia (NPA, PF ID: 1743), which is the name used for joint manifestos after the party's rebranding; manifestos are published under NPA, not NAT.",implemented,llm_verified
+1743,NPA,338,NAT,AU,implemented
-1760,HDZ BiH,3904,HDZ-HK~HNZ,BA,HDZ-led coalition variants,implemented,manual
+1760,HDZ BiH,3904,HDZ-HK~HNZ,BA,implemented
-36,N-VA,756,CD+NVA,BE,cartel list,implemented,manual
+36,N-VA,756,CD+NVA,BE,implemented
-604,CD/V,622,CD&V,BE,same party duplicate PF ID,implemented,manual
+604,CD/V,622,CD&V,BE,implemented
-604,CD/V,756,CD+NVA,BE,"LLM: CD+NVA refers to the joint electoral lists of CD&V and N-VA (mainly 2003-2007); during this period, they published joint manifestos under the CD/V (PF ID: 604) label in the text data.",implemented,llm_verified
+604,CD/V,756,CD+NVA,BE,implemented
-1680,sp.a,1586,sp.a-SPIRIT,BE,merger,implemented,manual
+1680,sp.a,1586,sp.a-SPIRIT,BE,implemented
-374,NDSV,5848,KSII,BG,"LLM: KSII is the coalition led by NDSV (374); manifestos published under NDSV, not KSII.",implemented,llm_verified
+374,NDSV,5848,KSII,BG,implemented
-482,SDS,3908,G-VMRO; VMRO-BND,BG,bloc constituent (progtype=8),implemented,progtype_8
+482,SDS,3908,G-VMRO; VMRO-BND,BG,implemented
-1765,ONS,3908,G-VMRO; VMRO-BND,BG,bloc constituent (progtype=8),implemented,progtype_8
+1765,ONS,3908,G-VMRO; VMRO-BND,BG,implemented
-5649,Patriotic Front - NFSB and VMRO,2057,NFSB,BG,bloc constituent (progtype=8),implemented,progtype_8
+5649,Patriotic Front - NFSB and VMRO,2057,NFSB,BG,implemented
-5649,Patriotic Front - NFSB and VMRO,3908,G-VMRO; VMRO-BND,BG,bloc constituent (progtype=8),implemented,progtype_8
+5649,Patriotic Front - NFSB and VMRO,3908,G-VMRO; VMRO-BND,BG,implemented
-360,FDP/PLR,1231,FDP/PLR,CH,same party duplicate PF ID,implemented,manual
+360,FDP/PLR,1231,FDP/PLR,CH,implemented
-6061,Alliance,928,RN,CL,core coalition member joint CMP manifesto,implemented,manual
+6061,Alliance,928,RN,CL,implemented
-6061,Alliance,1599,UDI,CL,core coalition member joint CMP manifesto,implemented,manual
+6061,Alliance,1599,UDI,CL,implemented
-1707,STAN,751,SNK-ED,CZ,"LLM: SNK-ED ran joint lists and published manifestos together with STAN (PF ID: 1707) in parliamentary elections, notably in 2006, under the SNK-ED–STAN label.",implemented,llm_verified
+1707,STAN,751,SNK-ED,CZ,implemented
-2138,LB,1041,KSC,CZ,bloc constituent (progtype=8),implemented,progtype_8
+2138,LB,1041,KSC,CZ,implemented
-6202,KDU-ČSL-US-DEU,104,US-DEU,CZ,bloc constituent (progtype=8),implemented,progtype_8
+6202,KDU-ČSL-US-DEU,104,US-DEU,CZ,implemented
-211,CDU/CSU,1375,CDU,DE,constituent,implemented,manual
+211,CDU/CSU,1375,CDU,DE,implemented
-211,CDU/CSU,1731,CSU,DE,constituent,implemented,manual
+211,CDU/CSU,1731,CSU,DE,implemented
-1816,B90/Grüne,10,Die Grünen,DE,merger,implemented,manual
+1816,B90/Grüne,10,Die Grünen,DE,implemented
-3925,RED-ID,797,ID,EC,constituent of alliance,implemented,manual
+3925,RED-ID,797,ID,EC,implemented
-685,RP,491,ERP,EE,LLM: ERP (Res Publica) published joint manifestos as 'RP' (PF ID: 685) in the text data; Res Publica is the Estonian name for the same party.,implemented,llm_verified
+685,RP,491,ERP,EE,implemented
-779,I/ERSP,908,RKI,EE,bloc constituent (progtype=8),implemented,progtype_8
+779,I/ERSP,908,RKI,EE,implemented
-779,I/ERSP,1299,ERSP,EE,merger,implemented,manual
+779,I/ERSP,1299,ERSP,EE,implemented
-139,CiU,4795,CDC,ES,constituent,implemented,manual
+139,CiU,4795,CDC,ES,implemented
-8271,Compromís–Podemos–EUPV,5623,CC,ES,LLM: CC (Compromís) published joint manifestos as part of Compromís–Podemos–EUPV (PF ID: 8271) in general elections.,implemented,llm_verified
+8271,Compromís–Podemos–EUPV,5623,CC,ES,implemented
-213,MoDem,496,MoDem,FR,same party duplicate PF ID,implemented,manual
+213,MoDem,496,MoDem,FR,implemented
-1108,EELV,5650,EELV,FR,same party duplicate PF ID,implemented,manual
+1108,EELV,5650,EELV,FR,implemented
-1595,UMP,4628,Les Républicains,FR,UMP renamed 2015,implemented,manual
+1595,UMP,4628,Les Républicains,FR,implemented
-1595,UMP,8168,LR,FR,same as Les Républicains duplicate PF ID,implemented,manual
+1595,UMP,8168,LR,FR,implemented
-1468,PASOK,7909,KINAL,GR,PASOK-dominated umbrella rebranded 2022,implemented,manual
+1468,PASOK,7909,KINAL,GR,implemented
-7347,EL,378,OP,GR,LLM: Oikologoi Prasinoi (OP) ran jointly with MeRA25 in 2019 as part of the 'MeRA25-Alliance for Breakup' and did not publish a separate manifesto; their program was subsumed under MeRA25.,implemented,llm_verified
+7347,EL,378,OP,GR,implemented
-1475,SDP,8842,SDP-HSLS,HR,joint list SDP dominant,implemented,manual
+1475,SDP,8842,SDP-HSLS,HR,implemented
-2522,Kukuriku,78,DC,HR,"LLM: DC (Democratic Centre) was a constituent of the Kukuriku coalition, which published joint manifestos under the Kukuriku name (PF ID: 2522) in 2011 and 2015.",implemented,llm_verified
+2522,Kukuriku,78,DC,HR,implemented
-3648,ZL,8036,HKDU,HR,bloc constituent (progtype=5),implemented,progtype_5
+3648,ZL,8036,HKDU,HR,implemented
-3918,DA-IDS-RDS,513,IDS,HR,constituent of alliance,implemented,manual
+3918,DA-IDS-RDS,513,IDS,HR,implemented
-242,PBP,8241,PBPS,IE,LLM: PBPS (Solidarity–People Before Profit) publishes joint manifestos under PBP (PF ID: 242) in the text data; this is a union.,implemented,llm_verified
+242,PBP,8241,PBPS,IE,implemented
-201,UdC,1758,UC,IT,LLM: Unione di Centro (UC/UDC) is a constituent of UdC (PF ID: 201) in the text data; they publish joint manifestos under UdC.,implemented,llm_verified
+201,UdC,1758,UC,IT,implemented
-1212,SEL,7031,SEL,IT,LLM: Sinistra Ecologia Libertà (SEL) is present in both datasets and publishes manifestos under SEL (PF ID: 1212) in text data.,implemented,llm_verified
+1212,SEL,7031,SEL,IT,implemented
-1737,Olive Tree,878,DS,IT,bloc constituent (progtype=8),implemented,progtype_8
+1737,Olive Tree,878,DS,IT,implemented
-6241,House of Freedom,813,AN,IT,bloc constituent (progtype=8),implemented,progtype_8
+6241,House of Freedom,813,AN,IT,implemented
-6241,House of Freedom,1519,CeD,IT,bloc constituent (progtype=8),implemented,progtype_8
+6241,House of Freedom,1519,CeD,IT,implemented
-1967,SLFP,4020,CP / VLSSP,LK,"LLM: CP/VLSSP often contested as part of the SLFP-led United Front and People's Alliance, typically under joint manifestos with SLFP, but sometimes ran separately; medium confidence due to occasional independent runs.",implemented,llm_verified
+1967,SLFP,4020,CP / VLSSP,LK,implemented
-1967,SLFP,5414,LSS,LK,"LLM: LSSP was a core constituent of the SLFP-led United Front and People's Alliance, publishing joint manifestos with SLFP.",implemented,llm_verified
+1967,SLFP,5414,LSS,LK,implemented
-1967,SLFP,6691,CP,LK,"LLM: CP was a constituent of the SLFP-led United Front and People's Alliance, publishing joint manifestos with SLFP.",implemented,llm_verified
+1967,SLFP,6691,CP,LK,implemented
-197,BSDK,168,LRS,LT,bloc constituent (progtype=8),implemented,progtype_8
+197,BSDK,168,LRS,LT,implemented
-197,BSDK,1747,LMP-NDP,LT,bloc constituent (progtype=8),implemented,progtype_8
+197,BSDK,1747,LMP-NDP,LT,implemented
-377,LTS,1410,LLaS,LT,bloc constituent (progtype=8),implemented,progtype_8
+377,LTS,1410,LLaS,LT,implemented
-5779,SK,1407,LZP,LT,bloc constituent (progtype=8),implemented,progtype_8
+5779,SK,1407,LZP,LT,implemented
-186,LSAP/POSL,898,SDP,LU,same party duplicate PF ID,implemented,manual
+186,LSAP/POSL,898,SDP,LU,implemented
-708,LNNK-LZP,1296,LZP,LV,name fragment of LNNK-LZP,implemented,name_fragment
+708,LNNK-LZP,1296,LZP,LV,implemented
-1704,TB-LNNK,671,TB,LV,constituent of alliance,implemented,manual
+1704,TB-LNNK,671,TB,LV,implemented
-1704,TB-LNNK,1789,LNNK,LV,constituent of alliance,implemented,manual
+1704,TB-LNNK,1789,LNNK,LV,implemented
-1704,TB-LNNK,7619,NATBLNNK,LV,"LLM: NATBLNNK is a constituent of TB-LNNK (1704), which published joint manifestos as a union.",implemented,llm_verified
+1704,TB-LNNK,7619,NATBLNNK,LV,implemented
-7622,ACUM,7904,PAS,MD,bloc constituent (progtype=8),implemented,progtype_8
+7622,ACUM,7904,PAS,MD,implemented
-3254,DSCG,3253,HGI,ME,LLM: HGI (Croatian Civic Initiative) is a small Croatian minority party that typically runs on joint lists with DSCG (Democratic Union of Croats) in Montenegro; manifestos are published under DSCG.,implemented,llm_verified
+3254,DSCG,3253,HGI,ME,implemented
-1537,GL,1533,Groen,NL,"LLM: Groen was a constituent party of GroenLinks (GL, PF ID: 1537), which published joint manifestos after its formation; Groen did not publish separate manifestos after joining the union.",implemented,llm_verified
+1537,GL,1533,Groen,NL,implemented
-716,Alliance,1119,NLP,NZ,LLM: NLP (NewLabour Party) was a founding constituent of the Alliance and published joint manifestos under the Alliance name.,implemented,llm_verified
+716,Alliance,1119,NLP,NZ,implemented
-4219,C90,5130,P2000,PE,"LLM: P2000 (Perú 2000) was a bloc including Cambio 90 (C90); manifestos were published under the C90/NM/P2000 bloc, which is represented as C90 (PF ID: 4219) in text data.",implemented,llm_verified
+4219,C90,5130,P2000,PE,implemented
-1458,WAK,70,ZChN,PL,bloc constituent (progtype=8),implemented,progtype_8
+1458,WAK,70,ZChN,PL,implemented
-8268,UW,1566,D|W|U,PL,"LLM: D|W|U refers to Unia Wolności (UW) and its predecessors/successors, which published manifestos under the UW name (PF ID: 8268) in the text data.",implemented,llm_verified
+8268,UW,1566,D|W|U,PL,implemented
-192,CDR,645,PAC,RO,bloc constituent (progtype=8),implemented,progtype_8
+192,CDR,645,PAC,RO,implemented
-1347,PSD-PUR,1443,PU|PC,RO,name fragment of PSD-PUR,implemented,name_fragment
+1347,PSD-PUR,1443,PU|PC,RO,implemented
-5941,USL,120,PSD,RO,PSD was the lead constituent of USL coalition (2012 joint manifesto),implemented,manual
+5941,USL,120,PSD,RO,implemented
-5941,USL,481,PNL,RO,PNL was a core constituent of USL coalition (2012 joint manifesto),implemented,manual
+5941,USL,481,PNL,RO,implemented
-5941,USL,1541,UNPR,RO,LLM: UNPR was a constituent of the USL (PF ID: 5941) coalition and did not publish its own manifesto; manifestos were issued under the USL name.,implemented,llm_verified
+5941,USL,1541,UNPR,RO,implemented
-6153,PSD-PC,1443,PU|PC,RO,name fragment of PSD-PC,implemented,name_fragment
+6153,PSD-PC,1443,PU|PC,RO,implemented
-8626,LDP/LSV/SDS,4769,LSV,RS,name fragment of LDP/LSV/SDS,implemented,name_fragment
+8626,LDP/LSV/SDS,4769,LSV,RS,implemented
-205,SV,200,SDSS,SK,bloc constituent (progtype=8),implemented,progtype_8
+205,SV,200,SDSS,SK,implemented
-226,SDK,200,SDSS,SK,bloc constituent (progtype=8),implemented,progtype_8
+226,SDK,200,SDSS,SK,implemented
-1617,SDKÚ-DS,983,DS,SK,DS merged into SDKÚ-DS,implemented,manual
+1617,SDKÚ-DS,983,DS,SK,implemented
-6629,DÚS,707,DUS,SK,LLM: DUS (Demokratická únia Slovenska) is the same as DÚS (PF ID: 6629) in the manifesto data; manifestos are published under the union name.,implemented,llm_verified
+6629,DÚS,707,DUS,SK,implemented
-1658,FA,3671,NE,UY,"LLM: NE (Nuevo Espacio) is a well-known constituent party of the Frente Amplio (FA) coalition, which publishes joint manifestos under the FA name.",implemented,llm_verified
+1658,FA,3671,NE,UY,implemented
-301,"SYRIZA, SYN; SYRIZA, Syriza, SYN/SYRIZA",1682,DIKKI,GR,"LLM (bloc-centric): DIKKI was a constituent member of the SYRIZA bloc in the 2007 election, running under its banner and publishing joint manifestos.",implemented,llm_verified
+301,"SYRIZA, SYN; SYRIZA, Syriza, SYN/SYRIZA",1682,DIKKI,GR,implemented
-676,"KDU, KDU-ČSL, KDU-CSL, KDU/CSL, KDUCSL, KDU–CSL, KDU–Č, KDU- ČSL, CSL",824,KDS,CZ,LLM (bloc-centric): KDS (Christian Democratic Party) is explicitly listed as a constituent member of the 'Christian and Democratic Union - Czech People's Party' bloc in the PartyFacts comments and published joint manifestos with it.,implemented,llm_verified
+676,"KDU, KDU-ČSL, KDU-CSL, KDU/CSL, KDUCSL, KDU–CSL, KDU–Č, KDU- ČSL, CSL",824,KDS,CZ,implemented
-701,"ZZS, LZS",1702,LZS,LV,"LLM (bloc-centric): Latvijas Zemnieku savienība (LZS) was a core constituent member of the Greens' and Farmers’ Union (ZZS) bloc in 2002, publishing joint manifestos and running under the bloc's banner.",implemented,llm_verified
+701,"ZZS, LZS",1702,LZS,LV,implemented
-852,"V, Unity, UNITY, VIENOTIBA, JV, PS",1531,JL,LV,LLM (bloc-centric): Jaunais laiks (JL) was a founding constituent of the Unity (Vienotība) bloc in 2010 and published joint manifestos under its banner.,implemented,llm_verified
+852,"V, Unity, UNITY, VIENOTIBA, JV, PS",1531,JL,LV,implemented
-2190,"DSS, DSS/NS",2346,NS,RS,"LLM (bloc-centric): New Serbia (NS) was a verified constituent member of the 'Democratic Party of Serbia/New Serbia' bloc in the 2008 elections, publishing joint manifestos with DSS.",implemented,llm_verified
+2190,"DSS, DSS/NS",2346,NS,RS,implemented
-2530,"FpV, FPV, FPV-PJ, AFplV, FplV",623,PJ,AR,"LLM (bloc-centric): The Justicialist Party (PJ) was the principal and founding constituent of the Front for Victory (FpV) bloc, running under its banner and publishing joint manifestos in all relevant elections.",implemented,llm_verified
+2530,"FpV, FPV, FPV-PJ, AFplV, FplV",623,PJ,AR,implemented
-3906,NA,356,PT,BR,"LLM (bloc-centric): PT (Partido dos Trabalhadores) was the leading party and consistent constituent of this left/progressive bloc across all listed elections, publishing joint manifestos under its banner.",implemented,llm_verified
+3906,NA,356,PT,BR,implemented
-3906,NA,723,PSB,BR,"LLM (bloc-centric): PSB (Partido Socialista Brasileiro) was a frequent coalition partner and constituent member of this bloc, including joint manifestos in several elections (notably 2002, 2006, 2010, 2014, and 2022).",implemented,llm_verified
+3906,NA,723,PSB,BR,implemented
-3906,NA,1009,PDT,BR,"LLM (bloc-centric): PDT (Partido Democrático Trabalhista) was a constituent member of this bloc in multiple elections, including joint manifestos (notably 2010, 2018, and 2022).",implemented,llm_verified
+3906,NA,1009,PDT,BR,implemented
-3906,NA,4405,"PR, PR (2), PR / PL, PR/PL, PR PL, PL/PR",BR,"LLM (bloc-centric): PR (Partido da República) was a constituent member of the bloc in the 2010 and 2014 elections, publishing joint manifestos with the bloc.",implemented,llm_verified
+3906,NA,4405,"PR, PR (2), PR / PL, PR/PL, PR PL, PL/PR",BR,implemented
-3906,NA,458,PTB,BR,"LLM (bloc-centric): PTB (Partido Trabalhista Brasileiro) was a constituent member of the bloc in the 2002 and 2006 elections, participating in joint manifestos.",implemented,llm_verified
+3906,NA,458,PTB,BR,implemented
-3906,NA,1823,PL,BR,"LLM (bloc-centric): PL (Partido Liberal) was a constituent member of the bloc in the 2002 election, publishing a joint manifesto.",implemented,llm_verified
+3906,NA,1823,PL,BR,implemented
-4550,"C, Concertacion, CPD",6,PS,CL,"LLM (bloc-centric): The Socialist Party of Chile (PS) was a founding and continuous member of the Concertación/CPD, running on joint lists and publishing joint manifestos.",implemented,llm_verified
+4550,"C, Concertacion, CPD",6,PS,CL,implemented
-4550,"C, Concertacion, CPD",54,PPD,CL,"LLM (bloc-centric): The Party for Democracy (PPD) was a core constituent of the Concertación/CPD, participating in all its joint electoral platforms and manifestos.",implemented,llm_verified
+4550,"C, Concertacion, CPD",54,PPD,CL,implemented
-4550,"C, Concertacion, CPD",390,PDC,CL,"LLM (bloc-centric): The Christian Democratic Party (PDC) was a principal founding member of the Concertación/CPD, running under its banner and signing joint manifestos.",implemented,llm_verified
+4550,"C, Concertacion, CPD",390,PDC,CL,implemented
-4550,"C, Concertacion, CPD",437,PRSD,CL,"LLM (bloc-centric): The Radical Social Democratic Party (PRSD) was a constituent member of the Concertación/CPD, participating in joint electoral lists and manifestos.",implemented,llm_verified
+4550,"C, Concertacion, CPD",437,PRSD,CL,implemented
-8999,"ZMS, Aleksandar V..., PS-TN, Serbia is Wi..., ally",3177,SNS,RS,LLM (bloc-centric): The Serbian Progressive Party (SNS) was the leading and founding constituent of the 'Aleksandar Vučić – Serbia wins / For Our Children / Serbia Must Not Stop' bloc in all listed elections and published joint manifestos under this banner.,implemented,llm_verified
+8999,"ZMS, Aleksandar V..., PS-TN, Serbia is Wi..., ally",3177,SNS,RS,implemented
-1117,PO,4630,.N,PL,Nowoczesna was core constituent of Koalicja Obywatelska (KO) in 2019 under PO CMP code (progtype=8),implemented,manual
+1117,PO,4630,.N,PL,implemented
-4550,Concertacion,162,PC,CL,Communist Party of Chile was constituent of Nueva Mayoría (2013-2017) under Concertación PF ID,implemented,manual
+4550,Concertacion,162,PC,CL,implemented
-4550,Concertacion,209,PH,CL,Humanist Party was Concertación constituent (2005-2009),implemented,manual
+4550,Concertacion,209,PH,CL,implemented
-5668,EH Bildu,1671,Amaiur,ES,Amaiur (2011) predecessor to EH Bildu; expert data 2014-2024 covers EH Bildu years,implemented,manual
+5668,EH Bildu,1671,Amaiur,ES,implemented
-6241,CdL,1767,CCD,IT,CdL coalition constituent (1994-2008),implemented,manual
+6241,CdL,1767,CCD,IT,implemented
-3979,Salvemos a México,1474,PRI,MX,PRI-PVEM electoral coalition (2006-2012) under various names,implemented,manual
+3979,Salvemos a México,1474,PRI,MX,implemented
-3979,Salvemos a México,446,PVEM,MX,PRI-PVEM electoral coalition (2006-2012) under various names,implemented,manual
+3979,Salvemos a México,446,PVEM,MX,implemented
-7912,Joint List,421,Hadash,IL,Arab party coalition (2015-2021); Hadash is core constituent,implemented,manual
+7912,Joint List,421,Hadash,IL,implemented
-7912,Joint List,1663,Balad,IL,Arab party coalition (2015-2021); Balad is constituent,implemented,manual
+7912,Joint List,1663,Balad,IL,implemented
-365,PdL,1626,FI,IT,merger constituent,implemented,manual
+365,PdL,1626,FI,IT,implemented
-365,PdL,813,AN,IT,merger constituent,implemented,manual
+365,PdL,813,AN,IT,implemented
@@ -171,7 +171,6 @@ alliance_union_harmonization <- bind_rows(
  tibble(metric = "unique_union_or_alliance_ids", category = "all", value = n_distinct(union_mapping$manifesto_pf_id)),
  tibble(metric = "unique_constituent_party_ids", category = "all", value = n_distinct(union_mapping$expert_pf_id)),
  union_mapping %>% count(country, name = "value") %>% transmute(metric = "mappings_by_country", category = country, value),
  union_mapping %>% count(relationship, name = "value") %>% transmute(metric = "mappings_by_relationship", category = relationship, value),
  union_mapping %>% count(status, name = "value") %>% transmute(metric = "mappings_by_status", category = status, value)
 )
@@ -189,11 +188,11 @@ if (is.na(convergence_summary_file) || is.na(convergence_detail_file)) {
  stop("Convergence diagnostics not found under ", outputs_dir, ". Run the model diagnostics before generating the report.")
 }
 model_convergence_summary <- read_if_exists(convergence_summary_file) %>%
-  mutate(source_file = convergence_summary_file)
+  identity()
 model_convergence_by_dimension <- read_if_exists(convergence_detail_file) %>%
  group_by(dimension) %>%
  summarise(parameters = n(), mean_rhat = mean(rhat, na.rm = TRUE), max_rhat = max(rhat, na.rm = TRUE), min_ess_bulk = min(ess_bulk, na.rm = TRUE), mean_ess_bulk = mean(ess_bulk, na.rm = TRUE), .groups = "drop") %>%
-  mutate(dimension = public_dimension(dimension), source_file = convergence_detail_file) %>%
+  mutate(dimension = public_dimension(dimension)) %>%
  arrange(dimension)
 convergent_summary_file <- latest_file(file.path(outputs_dir, "validation", "latest"), "^convergent_summary_.*\\.csv$")
@@ -216,14 +215,13 @@ uncertainty_summary <- read_if_exists(uncertainty_summary_file) %>%
 external_validation_correlations <- read_if_exists(external_validation_file) %>%
  group_by(var, dimension) %>%
  summarise(n = n(), pearson_r = cor(expert_val, model_val, use = "complete.obs"), mean_absolute_error = mean(abs_error, na.rm = TRUE), coverage_95 = mean(covered_95, na.rm = TRUE), .groups = "drop") %>%
-  mutate(dimension = public_dimension(dimension), source_file = external_validation_file) %>%
+  mutate(dimension = public_dimension(dimension)) %>%
  arrange(dimension, var)
 construct_family_positions <- read_if_exists(construct_families_file) %>%
  rename(mean_cultural = mean_galtan, sd_cultural = sd_galtan) %>%
  mutate(source_file = construct_families_file) %>%
  arrange(mean_economic)
 construct_temporal_stability <- read_if_exists(construct_unstable_file) %>%
-  mutate(dimension = public_dimension(dimension), source_file = construct_unstable_file) %>%
+  mutate(dimension = public_dimension(dimension)) %>%
  arrange(desc(annual_change))
 source_composition_balance <- read_if_exists(review_file("validation", "source_composition_balance.csv")) %>%
  mutate(dimension = public_dimension(dimension))
@@ -36,78 +36,4 @@ mappings_by_country,RO,6
 mappings_by_country,RS,3
 mappings_by_country,SK,4
 mappings_by_country,UY,1
 mappings_by_relationship,Amaiur (2011) predecessor to EH Bildu; expert data 2014-2024 covers EH Bildu years,1
 mappings_by_relationship,Arab party coalition (2015-2021); Balad is constituent,1
 mappings_by_relationship,Arab party coalition (2015-2021); Hadash is core constituent,1
 mappings_by_relationship,CdL coalition constituent (1994-2008),1
 mappings_by_relationship,Communist Party of Chile was constituent of Nueva Mayoría (2013-2017) under Concertación PF ID,1
 mappings_by_relationship,DS merged into SDKÚ-DS,1
 mappings_by_relationship,HDZ-led coalition variants,1
 mappings_by_relationship,Humanist Party was Concertación constituent (2005-2009),1
 mappings_by_relationship,"LLM (bloc-centric): DIKKI was a constituent member of the SYRIZA bloc in the 2007 election, running under its banner and publishing joint manifestos.",1
 mappings_by_relationship,LLM (bloc-centric): Jaunais laiks (JL) was a founding constituent of the Unity (Vienotība) bloc in 2010 and published joint manifestos under its banner.,1
 mappings_by_relationship,LLM (bloc-centric): KDS (Christian Democratic Party) is explicitly listed as a constituent member of the 'Christian and Democratic Union - Czech People's Party' bloc in the PartyFacts comments and published joint manifestos with it.,1
 mappings_by_relationship,"LLM (bloc-centric): Latvijas Zemnieku savienība (LZS) was a core constituent member of the Greens' and Farmers’ Union (ZZS) bloc in 2002, publishing joint manifestos and running under the bloc's banner.",1
 mappings_by_relationship,"LLM (bloc-centric): New Serbia (NS) was a verified constituent member of the 'Democratic Party of Serbia/New Serbia' bloc in the 2008 elections, publishing joint manifestos with DSS.",1
 mappings_by_relationship,"LLM (bloc-centric): PDT (Partido Democrático Trabalhista) was a constituent member of this bloc in multiple elections, including joint manifestos (notably 2010, 2018, and 2022).",1
 mappings_by_relationship,"LLM (bloc-centric): PL (Partido Liberal) was a constituent member of the bloc in the 2002 election, publishing a joint manifesto.",1
 mappings_by_relationship,"LLM (bloc-centric): PR (Partido da República) was a constituent member of the bloc in the 2010 and 2014 elections, publishing joint manifestos with the bloc.",1
 mappings_by_relationship,"LLM (bloc-centric): PSB (Partido Socialista Brasileiro) was a frequent coalition partner and constituent member of this bloc, including joint manifestos in several elections (notably 2002, 2006, 2010, 2014, and 2022).",1
 mappings_by_relationship,"LLM (bloc-centric): PT (Partido dos Trabalhadores) was the leading party and consistent constituent of this left/progressive bloc across all listed elections, publishing joint manifestos under its banner.",1
 mappings_by_relationship,"LLM (bloc-centric): PTB (Partido Trabalhista Brasileiro) was a constituent member of the bloc in the 2002 and 2006 elections, participating in joint manifestos.",1
 mappings_by_relationship,"LLM (bloc-centric): The Christian Democratic Party (PDC) was a principal founding member of the Concertación/CPD, running under its banner and signing joint manifestos.",1
 mappings_by_relationship,"LLM (bloc-centric): The Justicialist Party (PJ) was the principal and founding constituent of the Front for Victory (FpV) bloc, running under its banner and publishing joint manifestos in all relevant elections.",1
 mappings_by_relationship,"LLM (bloc-centric): The Party for Democracy (PPD) was a core constituent of the Concertación/CPD, participating in all its joint electoral platforms and manifestos.",1
 mappings_by_relationship,"LLM (bloc-centric): The Radical Social Democratic Party (PRSD) was a constituent member of the Concertación/CPD, participating in joint electoral lists and manifestos.",1
 mappings_by_relationship,LLM (bloc-centric): The Serbian Progressive Party (SNS) was the leading and founding constituent of the 'Aleksandar Vučić – Serbia wins / For Our Children / Serbia Must Not Stop' bloc in all listed elections and published joint manifestos under this banner.,1
 mappings_by_relationship,"LLM (bloc-centric): The Socialist Party of Chile (PS) was a founding and continuous member of the Concertación/CPD, running on joint lists and publishing joint manifestos.",1
 mappings_by_relationship,LLM: CC (Compromís) published joint manifestos as part of Compromís–Podemos–EUPV (PF ID: 8271) in general elections.,1
 mappings_by_relationship,"LLM: CD+NVA refers to the joint electoral lists of CD&V and N-VA (mainly 2003-2007); during this period, they published joint manifestos under the CD/V (PF ID: 604) label in the text data.",1
 mappings_by_relationship,"LLM: CP was a constituent of the SLFP-led United Front and People's Alliance, publishing joint manifestos with SLFP.",1
 mappings_by_relationship,"LLM: CP/VLSSP often contested as part of the SLFP-led United Front and People's Alliance, typically under joint manifestos with SLFP, but sometimes ran separately; medium confidence due to occasional independent runs.",1
 mappings_by_relationship,"LLM: DC (Democratic Centre) was a constituent of the Kukuriku coalition, which published joint manifestos under the Kukuriku name (PF ID: 2522) in 2011 and 2015.",1
 mappings_by_relationship,LLM: DUS (Demokratická únia Slovenska) is the same as DÚS (PF ID: 6629) in the manifesto data; manifestos are published under the union name.,1
 mappings_by_relationship,"LLM: D|W|U refers to Unia Wolności (UW) and its predecessors/successors, which published manifestos under the UW name (PF ID: 8268) in the text data.",1
 mappings_by_relationship,LLM: ERP (Res Publica) published joint manifestos as 'RP' (PF ID: 685) in the text data; Res Publica is the Estonian name for the same party.,1
 mappings_by_relationship,"LLM: FPCyS (Frente Progresista, Cívico y Social) included PS and others; manifestos often under FAP or similar coalitions.",1
 mappings_by_relationship,"LLM: FR (Frente Renovador) joined FAP in some elections, but also ran independently; likely constituent in FAP manifestos.",1
 mappings_by_relationship,"LLM: Groen was a constituent party of GroenLinks (GL, PF ID: 1537), which published joint manifestos after its formation; Groen did not publish separate manifestos after joining the union.",1
 mappings_by_relationship,LLM: HGI (Croatian Civic Initiative) is a small Croatian minority party that typically runs on joint lists with DSCG (Democratic Union of Croats) in Montenegro; manifestos are published under DSCG.,1
 mappings_by_relationship,"LLM: KSII is the coalition led by NDSV (374); manifestos published under NDSV, not KSII.",1
 mappings_by_relationship,"LLM: LSSP was a core constituent of the SLFP-led United Front and People's Alliance, publishing joint manifestos with SLFP.",1
 mappings_by_relationship,"LLM: NATBLNNK is a constituent of TB-LNNK (1704), which published joint manifestos as a union.",1
 mappings_by_relationship,"LLM: NE (Nuevo Espacio) is a well-known constituent party of the Frente Amplio (FA) coalition, which publishes joint manifestos under the FA name.",1
 mappings_by_relationship,LLM: NLP (NewLabour Party) was a founding constituent of the Alliance and published joint manifestos under the Alliance name.,1
 mappings_by_relationship,LLM: Oikologoi Prasinoi (OP) ran jointly with MeRA25 in 2019 as part of the 'MeRA25-Alliance for Breakup' and did not publish a separate manifesto; their program was subsumed under MeRA25.,1
 mappings_by_relationship,"LLM: P2000 (Perú 2000) was a bloc including Cambio 90 (C90); manifestos were published under the C90/NM/P2000 bloc, which is represented as C90 (PF ID: 4219) in text data.",1
 mappings_by_relationship,LLM: PBPS (Solidarity–People Before Profit) publishes joint manifestos under PBP (PF ID: 242) in the text data; this is a union.,1
 mappings_by_relationship,"LLM: PS (Socialist Party) was a core constituent of FAP (Frente Amplio Progresista), which published joint manifestos.",1
 mappings_by_relationship,"LLM: SNK-ED ran joint lists and published manifestos together with STAN (PF ID: 1707) in parliamentary elections, notably in 2006, under the SNK-ED–STAN label.",1
 mappings_by_relationship,LLM: Sinistra Ecologia Libertà (SEL) is present in both datasets and publishes manifestos under SEL (PF ID: 1212) in text data.,1
 mappings_by_relationship,"LLM: The Liberal Party (LP, PF ID: 1998) is the predecessor and constituent of the Liberal Party of Australia (LPA, PF ID: 486); manifestos are published under LPA, not LP.",1
 mappings_by_relationship,"LLM: The National Party (NAT) is the predecessor and constituent of the National Party of Australia (NPA, PF ID: 1743), which is the name used for joint manifestos after the party's rebranding; manifestos are published under NPA, not NAT.",1
 mappings_by_relationship,LLM: UNPR was a constituent of the USL (PF ID: 5941) coalition and did not publish its own manifesto; manifestos were issued under the USL name.,1
 mappings_by_relationship,LLM: Unione di Centro (UC/UDC) is a constituent of UdC (PF ID: 201) in the text data; they publish joint manifestos under UdC.,1
 mappings_by_relationship,Nowoczesna was core constituent of Koalicja Obywatelska (KO) in 2019 under PO CMP code (progtype=8),1
 mappings_by_relationship,PASOK-dominated umbrella rebranded 2022,1
 mappings_by_relationship,PNL was a core constituent of USL coalition (2012 joint manifesto),1
 mappings_by_relationship,PRI-PVEM electoral coalition (2006-2012) under various names,2
 mappings_by_relationship,PSD was the lead constituent of USL coalition (2012 joint manifesto),1
 mappings_by_relationship,Peronist faction no independent manifesto,1
 mappings_by_relationship,UMP renamed 2015,1
 mappings_by_relationship,bloc constituent (progtype=5),1
 mappings_by_relationship,bloc constituent (progtype=8),19
 mappings_by_relationship,cartel list,1
 mappings_by_relationship,constituent,3
 mappings_by_relationship,constituent of alliance,4
 mappings_by_relationship,core coalition member joint CMP manifesto,2
 mappings_by_relationship,joint list SDP dominant,1
 mappings_by_relationship,merger,3
 mappings_by_relationship,merger constituent,2
 mappings_by_relationship,name fragment of LDP/LSV/SDS,1
 mappings_by_relationship,name fragment of LNNK-LZP,1
 mappings_by_relationship,name fragment of PSD-PC,1
 mappings_by_relationship,name fragment of PSD-PUR,1
 mappings_by_relationship,same as Les Républicains duplicate PF ID,1
 mappings_by_relationship,same party duplicate PF ID,5
 mappings_by_status,implemented,106
@@ -1,8 +1,8 @@
-family,n_parties,n_obs,mean_economic,sd_economic,mean_cultural,sd_cultural,family_name,source_file
+family,n_parties,n_obs,mean_economic,sd_economic,mean_cultural,sd_cultural,family_name
-com,49,1241,0.12001647469327872,0.0901861773223108,0.3834027613298184,0.18041637351915937,Communist/Far Left,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_families_2026-05-04_18-12-36.csv
+com,49,1241,0.12001647469327872,0.0901861773223108,0.3834027613298184,0.18041637351915937,Communist/Far Left
-eco,30,723,0.26720980626115975,0.11467133878653364,0.2514849926574344,0.09223964864606182,Green/Ecological,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_families_2026-05-04_18-12-36.csv
+eco,30,723,0.26720980626115975,0.11467133878653364,0.2514849926574344,0.09223964864606182,Green/Ecological
-soc,86,2892,0.3305567447692131,0.1240506242334158,0.3777170994931328,0.13783788204153472,Social Democratic,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_families_2026-05-04_18-12-36.csv
+soc,86,2892,0.3305567447692131,0.1240506242334158,0.3777170994931328,0.13783788204153472,Social Democratic
-chr,41,1596,0.5961213268671609,0.11930537492981286,0.5293978691891922,0.13964161332639527,Christian Democratic,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_families_2026-05-04_18-12-36.csv
+chr,41,1596,0.5961213268671609,0.11930537492981286,0.5293978691891922,0.13964161332639527,Christian Democratic
-right,50,975,0.6222552715406671,0.18991001824381296,0.720426765976975,0.15147355617267264,Radical Right,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_families_2026-05-04_18-12-36.csv
+right,50,975,0.6222552715406671,0.18991001824381296,0.720426765976975,0.15147355617267264,Radical Right
-con,82,2425,0.6519453485719768,0.17791542785462533,0.5375091531921928,0.14258881749137706,Conservative,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_families_2026-05-04_18-12-36.csv
+con,82,2425,0.6519453485719768,0.17791542785462533,0.5375091531921928,0.14258881749137706,Conservative
-lib,81,2063,0.6569946895012412,0.15224833699588255,0.37069062594934565,0.13158056834168683,Liberal,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_families_2026-05-04_18-12-36.csv
+lib,81,2063,0.6569946895012412,0.15224833699588255,0.37069062594934565,0.13158056834168683,Liberal
@@ -1,60 +1,60 @@
-party_id,country,dimension,year_from,year_to,val_from,val_to,change,annual_change,source_file
+party_id,country,dimension,year_from,year_to,val_from,val_to,change,annual_change
-556,LT,cultural cosmopolitan--traditionalist,2019,2020,0.77107951125,0.5289456789999999,0.24213383225000007,0.24213383225000007,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+556,LT,cultural cosmopolitan--traditionalist,2019,2020,0.77107951125,0.5289456789999999,0.24213383225000007,0.24213383225000007
-1663,IL,cultural cosmopolitan--traditionalist,2021,2022,0.1423029388125,0.35615504375,0.2138521049375,0.2138521049375,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1663,IL,cultural cosmopolitan--traditionalist,2021,2022,0.1423029388125,0.35615504375,0.2138521049375,0.2138521049375
-455,IL,cultural cosmopolitan--traditionalist,1997,1998,0.456850958125,0.6432782493750001,0.18642729125000007,0.18642729125000007,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+455,IL,cultural cosmopolitan--traditionalist,1997,1998,0.456850958125,0.6432782493750001,0.18642729125000007,0.18642729125000007
-455,IL,cultural cosmopolitan--traditionalist,1996,1997,0.27663313025,0.456850958125,0.180217827875,0.180217827875,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+455,IL,cultural cosmopolitan--traditionalist,1996,1997,0.27663313025,0.456850958125,0.180217827875,0.180217827875
-8393,LV,cultural cosmopolitan--traditionalist,2018,2019,0.4386382122500001,0.2627366591625,0.17590155308750005,0.17590155308750005,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+8393,LV,cultural cosmopolitan--traditionalist,2018,2019,0.4386382122500001,0.2627366591625,0.17590155308750005,0.17590155308750005
-281,BE,cultural cosmopolitan--traditionalist,1977,1978,0.6706878695,0.843416257375,0.172728387875,0.172728387875,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+281,BE,cultural cosmopolitan--traditionalist,1977,1978,0.6706878695,0.843416257375,0.172728387875,0.172728387875
-556,LT,cultural cosmopolitan--traditionalist,2001,2002,0.318926108375,0.49125159325,0.172325484875,0.172325484875,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+556,LT,cultural cosmopolitan--traditionalist,2001,2002,0.318926108375,0.49125159325,0.172325484875,0.172325484875
-964,IS,economic left-right,2016,2017,0.691268056,0.520838928125,0.17042912787499995,0.17042912787499995,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+964,IS,economic left-right,2016,2017,0.691268056,0.520838928125,0.17042912787499995,0.17042912787499995
-298,NL,cultural cosmopolitan--traditionalist,2018,2019,0.760919900625,0.592874692125,0.16804520850000004,0.16804520850000004,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+298,NL,cultural cosmopolitan--traditionalist,2018,2019,0.760919900625,0.592874692125,0.16804520850000004,0.16804520850000004
-901,FI,economic left-right,1992,1993,0.486145510375,0.64721287575,0.161067365375,0.161067365375,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+901,FI,economic left-right,1992,1993,0.486145510375,0.64721287575,0.161067365375,0.161067365375
-467,SI,cultural cosmopolitan--traditionalist,2018,2019,0.563748886625,0.4041139849999999,0.15963490162500005,0.15963490162500005,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+467,SI,cultural cosmopolitan--traditionalist,2018,2019,0.563748886625,0.4041139849999999,0.15963490162500005,0.15963490162500005
-1221,IT,economic left-right,2007,2008,0.559621162375,0.400493563,0.15912759937500004,0.15912759937500004,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1221,IT,economic left-right,2007,2008,0.559621162375,0.400493563,0.15912759937500004,0.15912759937500004
-901,FI,economic left-right,1991,1992,0.327277968125,0.486145510375,0.15886754225,0.15886754225,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+901,FI,economic left-right,1991,1992,0.327277968125,0.486145510375,0.15886754225,0.15886754225
-455,IL,cultural cosmopolitan--traditionalist,1998,1999,0.6432782493750001,0.7970417803750001,0.15376353099999995,0.15376353099999995,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+455,IL,cultural cosmopolitan--traditionalist,1998,1999,0.6432782493750001,0.7970417803750001,0.15376353099999995,0.15376353099999995
-1221,IT,economic left-right,2006,2007,0.7086087693750001,0.559621162375,0.14898760700000002,0.14898760700000002,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1221,IT,economic left-right,2006,2007,0.7086087693750001,0.559621162375,0.14898760700000002,0.14898760700000002
-2211,UA,economic left-right,2006,2007,0.416653024,0.5619799204999999,0.1453268964999999,0.1453268964999999,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+2211,UA,economic left-right,2006,2007,0.416653024,0.5619799204999999,0.1453268964999999,0.1453268964999999
-631,CH,economic left-right,2018,2019,0.613235049875,0.7569515025,0.143716452625,0.143716452625,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+631,CH,economic left-right,2018,2019,0.613235049875,0.7569515025,0.143716452625,0.143716452625
-298,NL,cultural cosmopolitan--traditionalist,2019,2020,0.592874692125,0.734536642,0.14166194987500005,0.14166194987500005,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+298,NL,cultural cosmopolitan--traditionalist,2019,2020,0.592874692125,0.734536642,0.14166194987500005,0.14166194987500005
-556,LT,cultural cosmopolitan--traditionalist,2000,2001,0.180505337875,0.318926108375,0.1384207705,0.1384207705,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+556,LT,cultural cosmopolitan--traditionalist,2000,2001,0.180505337875,0.318926108375,0.1384207705,0.1384207705
-1417,IL,cultural cosmopolitan--traditionalist,1968,1969,0.49841048887499995,0.635275993875,0.13686550500000003,0.13686550500000003,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1417,IL,cultural cosmopolitan--traditionalist,1968,1969,0.49841048887499995,0.635275993875,0.13686550500000003,0.13686550500000003
-81,ES,cultural cosmopolitan--traditionalist,1999,2000,0.44213168437499994,0.30819645050000005,0.1339352338749999,0.1339352338749999,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+81,ES,cultural cosmopolitan--traditionalist,1999,2000,0.44213168437499994,0.30819645050000005,0.1339352338749999,0.1339352338749999
-1417,IL,cultural cosmopolitan--traditionalist,1967,1968,0.364764852,0.49841048887499995,0.13364563687499997,0.13364563687499997,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1417,IL,cultural cosmopolitan--traditionalist,1967,1968,0.364764852,0.49841048887499995,0.13364563687499997,0.13364563687499997
-901,FI,economic left-right,1993,1994,0.64721287575,0.78024709825,0.13303422250000008,0.13303422250000008,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+901,FI,economic left-right,1993,1994,0.64721287575,0.78024709825,0.13303422250000008,0.13303422250000008
-409,SE,economic left-right,2018,2019,0.4993397851250001,0.6246204093750001,0.12528062425000003,0.12528062425000003,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+409,SE,economic left-right,2018,2019,0.4993397851250001,0.6246204093750001,0.12528062425000003,0.12528062425000003
-48,GR,cultural cosmopolitan--traditionalist,1999,2000,0.471858772375,0.594323158,0.122464385625,0.122464385625,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+48,GR,cultural cosmopolitan--traditionalist,1999,2000,0.471858772375,0.594323158,0.122464385625,0.122464385625
-212,DK,cultural cosmopolitan--traditionalist,2014,2015,0.339423171125,0.459806116125,0.12038294500000002,0.12038294500000002,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+212,DK,cultural cosmopolitan--traditionalist,2014,2015,0.339423171125,0.459806116125,0.12038294500000002,0.12038294500000002
-2415,IT,cultural cosmopolitan--traditionalist,2006,2007,0.6143681051250001,0.49546903375,0.11889907137500004,0.11889907137500004,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+2415,IT,cultural cosmopolitan--traditionalist,2006,2007,0.6143681051250001,0.49546903375,0.11889907137500004,0.11889907137500004
-1369,IT,cultural cosmopolitan--traditionalist,2013,2014,0.7406959332499999,0.6231419237500002,0.11755400949999972,0.11755400949999972,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1369,IT,cultural cosmopolitan--traditionalist,2013,2014,0.7406959332499999,0.6231419237500002,0.11755400949999972,0.11755400949999972
-5852,IS,cultural cosmopolitan--traditionalist,2018,2019,0.336059923125,0.45280653625,0.116746613125,0.116746613125,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+5852,IS,cultural cosmopolitan--traditionalist,2018,2019,0.336059923125,0.45280653625,0.116746613125,0.116746613125
-1417,IL,cultural cosmopolitan--traditionalist,1966,1967,0.2487840847,0.364764852,0.11598076729999995,0.11598076729999995,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1417,IL,cultural cosmopolitan--traditionalist,1966,1967,0.2487840847,0.364764852,0.11598076729999995,0.11598076729999995
-828,NL,cultural cosmopolitan--traditionalist,2019,2020,0.399398152875,0.5144400794999999,0.11504192662499996,0.11504192662499996,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+828,NL,cultural cosmopolitan--traditionalist,2019,2020,0.399398152875,0.5144400794999999,0.11504192662499996,0.11504192662499996
-2415,IT,cultural cosmopolitan--traditionalist,2007,2008,0.49546903375,0.3804985986625,0.1149704350875,0.1149704350875,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+2415,IT,cultural cosmopolitan--traditionalist,2007,2008,0.49546903375,0.3804985986625,0.1149704350875,0.1149704350875
-828,NL,cultural cosmopolitan--traditionalist,2020,2021,0.5144400794999999,0.6280684987500001,0.11362841925000022,0.11362841925000022,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+828,NL,cultural cosmopolitan--traditionalist,2020,2021,0.5144400794999999,0.6280684987500001,0.11362841925000022,0.11362841925000022
-573,DE,cultural cosmopolitan--traditionalist,2024,2025,0.34241723825000003,0.4558404575,0.11342321924999998,0.11342321924999998,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+573,DE,cultural cosmopolitan--traditionalist,2024,2025,0.34241723825000003,0.4558404575,0.11342321924999998,0.11342321924999998
-1424,BE,economic left-right,1977,1978,0.7125746831249999,0.825810219125,0.11323553600000004,0.11323553600000004,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1424,BE,economic left-right,1977,1978,0.7125746831249999,0.825810219125,0.11323553600000004,0.11323553600000004
-1173,NO,cultural cosmopolitan--traditionalist,2018,2019,0.353365422125,0.46349993075,0.11013450862499996,0.11013450862499996,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1173,NO,cultural cosmopolitan--traditionalist,2018,2019,0.353365422125,0.46349993075,0.11013450862499996,0.11013450862499996
-1651,GR,economic left-right,2013,2014,0.441388039625,0.551427035875,0.11003899624999997,0.11003899624999997,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1651,GR,economic left-right,2013,2014,0.441388039625,0.551427035875,0.11003899624999997,0.11003899624999997
-1660,GR,cultural cosmopolitan--traditionalist,2012,2013,0.70537148075,0.8146704603749999,0.1092989796249999,0.1092989796249999,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1660,GR,cultural cosmopolitan--traditionalist,2012,2013,0.70537148075,0.8146704603749999,0.1092989796249999,0.1092989796249999
-433,FR,economic left-right,2018,2019,0.433590134625,0.542142367875,0.10855223325000002,0.10855223325000002,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+433,FR,economic left-right,2018,2019,0.433590134625,0.542142367875,0.10855223325000002,0.10855223325000002
-1002,GB,cultural cosmopolitan--traditionalist,2014,2015,0.3184758865,0.2101810030625,0.10829488343750002,0.10829488343750002,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1002,GB,cultural cosmopolitan--traditionalist,2014,2015,0.3184758865,0.2101810030625,0.10829488343750002,0.10829488343750002
-623,AR,economic left-right,1989,1990,0.4145806991249999,0.5228341057499999,0.10825340662499994,0.10825340662499994,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+623,AR,economic left-right,1989,1990,0.4145806991249999,0.5228341057499999,0.10825340662499994,0.10825340662499994
-1305,RO,economic left-right,2000,2001,0.553705481625,0.446037918375,0.10766756325,0.10766756325,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1305,RO,economic left-right,2000,2001,0.553705481625,0.446037918375,0.10766756325,0.10766756325
-298,NL,cultural cosmopolitan--traditionalist,2020,2021,0.734536642,0.84215486075,0.10761821875,0.10761821875,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+298,NL,cultural cosmopolitan--traditionalist,2020,2021,0.734536642,0.84215486075,0.10761821875,0.10761821875
-1359,PT,economic left-right,2004,2005,0.495169041875,0.602184326875,0.10701528500000002,0.10701528500000002,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1359,PT,economic left-right,2004,2005,0.495169041875,0.602184326875,0.10701528500000002,0.10701528500000002
-1651,GR,economic left-right,2012,2013,0.3355888995,0.441388039625,0.10579914012500002,0.10579914012500002,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1651,GR,economic left-right,2012,2013,0.3355888995,0.441388039625,0.10579914012500002,0.10579914012500002
-623,AR,economic left-right,1990,1991,0.5228341057499999,0.62815575375,0.1053216480000001,0.1053216480000001,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+623,AR,economic left-right,1990,1991,0.5228341057499999,0.62815575375,0.1053216480000001,0.1053216480000001
-1305,RO,economic left-right,2001,2002,0.446037918375,0.3412955855,0.104742332875,0.104742332875,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1305,RO,economic left-right,2001,2002,0.446037918375,0.3412955855,0.104742332875,0.104742332875
-455,IL,cultural cosmopolitan--traditionalist,1991,1992,0.5368312063749999,0.432172542875,0.10465866349999992,0.10465866349999992,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+455,IL,cultural cosmopolitan--traditionalist,1991,1992,0.5368312063749999,0.432172542875,0.10465866349999992,0.10465866349999992
-2415,IT,cultural cosmopolitan--traditionalist,2008,2009,0.3804985986625,0.2762634036875,0.104235194975,0.104235194975,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+2415,IT,cultural cosmopolitan--traditionalist,2008,2009,0.3804985986625,0.2762634036875,0.104235194975,0.104235194975
-599,AT,economic left-right,2008,2009,0.4633838822500001,0.567595171125,0.10421128887499996,0.10421128887499996,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+599,AT,economic left-right,2008,2009,0.4633838822500001,0.567595171125,0.10421128887499996,0.10421128887499996
-669,CH,cultural cosmopolitan--traditionalist,2016,2017,0.514819538625,0.410640874875,0.10417866374999996,0.10417866374999996,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+669,CH,cultural cosmopolitan--traditionalist,2016,2017,0.514819538625,0.410640874875,0.10417866374999996,0.10417866374999996
-5852,IS,cultural cosmopolitan--traditionalist,2017,2018,0.23270289265,0.336059923125,0.10335703047499996,0.10335703047499996,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+5852,IS,cultural cosmopolitan--traditionalist,2017,2018,0.23270289265,0.336059923125,0.10335703047499996,0.10335703047499996
-669,CH,cultural cosmopolitan--traditionalist,2015,2016,0.617986882875,0.514819538625,0.10316734425000008,0.10316734425000008,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+669,CH,cultural cosmopolitan--traditionalist,2015,2016,0.617986882875,0.514819538625,0.10316734425000008,0.10316734425000008
-48,GR,cultural cosmopolitan--traditionalist,2010,2011,0.569647428,0.6723034049999999,0.10265597699999984,0.10265597699999984,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+48,GR,cultural cosmopolitan--traditionalist,2010,2011,0.569647428,0.6723034049999999,0.10265597699999984,0.10265597699999984
-1221,IT,economic left-right,2008,2009,0.400493563,0.50303734575,0.10254378275000003,0.10254378275000003,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1221,IT,economic left-right,2008,2009,0.400493563,0.50303734575,0.10254378275000003,0.10254378275000003
-1651,GR,economic left-right,2014,2015,0.551427035875,0.6535508147500001,0.10212377887500013,0.10212377887500013,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1651,GR,economic left-right,2014,2015,0.551427035875,0.6535508147500001,0.10212377887500013,0.10212377887500013
-1221,IT,economic left-right,2009,2010,0.50303734575,0.604409204875,0.10137185912500002,0.10137185912500002,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+1221,IT,economic left-right,2009,2010,0.50303734575,0.604409204875,0.10137185912500002,0.10137185912500002
-975,SI,economic left-right,1990,1991,0.579305296625,0.6803467895000002,0.1010414928750002,0.1010414928750002,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+975,SI,economic left-right,1990,1991,0.579305296625,0.6803467895000002,0.1010414928750002,0.1010414928750002
-338,AU,economic left-right,1992,1993,0.791994626125,0.6916490538750002,0.10034557224999983,0.10034557224999983,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/construct_unstable_2026-05-04_18-12-36.csv
+338,AU,economic left-right,1992,1993,0.791994626125,0.6916490538750002,0.10034557224999983,0.10034557224999983
@@ -1,11 +1,11 @@
-var,dimension,n,pearson_r,mean_absolute_error,coverage_95,source_file
+var,dimension,n,pearson_r,mean_absolute_error,coverage_95
-culsup_vparty,cultural cosmopolitan--traditionalist,536,0.8121247297636631,0.12821713597308768,0.3843283582089552,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/external_validation_2026-03-28_18-37-35.csv
+culsup_vparty,cultural cosmopolitan--traditionalist,536,0.8121247297636631,0.12821713597308768,0.3843283582089552
-galtan_ches,cultural cosmopolitan--traditionalist,222,0.9588005926603757,0.07875607868037135,0.5225225225225225,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/external_validation_2026-03-28_18-37-35.csv
+galtan_ches,cultural cosmopolitan--traditionalist,222,0.9588005926603757,0.07875607868037135,0.5225225225225225
-gender_vparty,cultural cosmopolitan--traditionalist,545,0.5626122704047269,0.16947520363543578,0.28990825688073396,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/external_validation_2026-03-28_18-37-35.csv
+gender_vparty,cultural cosmopolitan--traditionalist,545,0.5626122704047269,0.16947520363543578,0.28990825688073396
-immig_vparty,cultural cosmopolitan--traditionalist,537,0.7429394940511392,0.10311559715251396,0.4897579143389199,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/external_validation_2026-03-28_18-37-35.csv
+immig_vparty,cultural cosmopolitan--traditionalist,537,0.7429394940511392,0.10311559715251396,0.4897579143389199
-lgbt_vparty,cultural cosmopolitan--traditionalist,541,0.7941178030023652,0.09410224229993068,0.5508317929759704,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/external_validation_2026-03-28_18-37-35.csv
+lgbt_vparty,cultural cosmopolitan--traditionalist,541,0.7941178030023652,0.09410224229993068,0.5508317929759704
-relig_vparty,cultural cosmopolitan--traditionalist,548,0.6757229503671286,0.30309927660661495,0.04744525547445255,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/external_validation_2026-03-28_18-37-35.csv
+relig_vparty,cultural cosmopolitan--traditionalist,548,0.6757229503671286,0.30309927660661495,0.04744525547445255
-lrecon_ches,economic left-right,223,0.9739626905522167,0.05518814853885153,0.8116591928251121,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/external_validation_2026-03-28_18-37-35.csv
+lrecon_ches,economic left-right,223,0.9739626905522167,0.05518814853885153,0.8116591928251121
-lrecon_poppa,economic left-right,74,0.9799670973969279,0.0660246477855859,0.6621621621621622,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/external_validation_2026-03-28_18-37-35.csv
+lrecon_poppa,economic left-right,74,0.9799670973969279,0.0660246477855859,0.6621621621621622
-lrecon_vparty,economic left-right,534,0.8664105550524236,0.08828332773956499,0.6741573033707865,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/external_validation_2026-03-28_18-37-35.csv
+lrecon_vparty,economic left-right,534,0.8664105550524236,0.08828332773956499,0.6741573033707865
-welf_vparty,economic left-right,534,0.6821895613302613,0.17587920065205523,0.36329588014981273,/srv/projects/party4d/archive/party2d_replication/outputs/validation/latest/external_validation_2026-03-28_18-37-35.csv
+welf_vparty,economic left-right,534,0.6821895613302613,0.17587920065205523,0.36329588014981273
@@ -1,9 +1,9 @@
-dimension,parameters,mean_rhat,max_rhat,min_ess_bulk,mean_ess_bulk,source_file
+dimension,parameters,mean_rhat,max_rhat,min_ess_bulk,mean_ess_bulk
-cultural cosmopolitan--traditionalist,17585,1.0006424726753271,1.0063876592524996,810.5088684922246,6889.24553993031,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_diagnostics_2026-06-12_12-53-24.csv
+cultural cosmopolitan--traditionalist,17585,1.0006424726753271,1.0063876592524996,810.5088684922246,6889.24553993031
-economic left-right,17585,1.0003816350246089,1.0041031832256586,670.7084347577414,5573.455663381927,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_diagnostics_2026-06-12_12-53-24.csv
+economic left-right,17585,1.0003816350246089,1.0041031832256586,670.7084347577414,5573.455663381927
-lr_country_offset,65,1.0007983450724103,1.0042741902159762,1078.1975001602086,5434.867801353405,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_diagnostics_2026-06-12_12-53-24.csv
+lr_country_offset,65,1.0007983450724103,1.0042741902159762,1078.1975001602086,5434.867801353405
-lr_decade_offset,8,1.0003240812347383,1.0011721739400503,2069.6763143867825,3372.015629437053,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_diagnostics_2026-06-12_12-53-24.csv
+lr_decade_offset,8,1.0003240812347383,1.0011721739400503,2069.6763143867825,3372.015629437053
-lr_sigma,3,1.0010806271267045,1.001817706171186,1956.6110527635497,2582.401306906664,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_diagnostics_2026-06-12_12-53-24.csv
+lr_sigma,3,1.0010806271267045,1.001817706171186,1956.6110527635497,2582.401306906664
-lr_source_offset,3,1.0001329930739762,1.0002839044194227,3477.5194916377077,3814.373810300655,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_diagnostics_2026-06-12_12-53-24.csv
+lr_source_offset,3,1.0001329930739762,1.0002839044194227,3477.5194916377077,3814.373810300655
-lr_weight,3,1.0023083674707072,1.0023587853631075,1540.8061256188755,1560.2173784478186,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_diagnostics_2026-06-12_12-53-24.csv
+lr_weight,3,1.0023083674707072,1.0023587853631075,1540.8061256188755,1560.2173784478186
-mean_sigma,6,1.00453801817315,1.0092946499280384,534.8105812273362,918.5017536375844,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_diagnostics_2026-06-12_12-53-24.csv
+mean_sigma,6,1.00453801817315,1.0092946499280384,534.8105812273362,918.5017536375844
@@ -1,7 +1,7 @@
-metric,count,percentage,source_file
+metric,count,percentage
-R-hat < 1.01,35258,100,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_summary_2026-06-12_12-53-24.csv
+R-hat < 1.01,35258,100
-R-hat 1.01-1.05,0,0,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_summary_2026-06-12_12-53-24.csv
+R-hat 1.01-1.05,0,0
-R-hat > 1.05,0,0,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_summary_2026-06-12_12-53-24.csv
+R-hat > 1.05,0,0
-ESS > 1000,34853,98.85,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_summary_2026-06-12_12-53-24.csv
+ESS > 1000,34853,98.85
-ESS 400-1000,405,1.15,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_summary_2026-06-12_12-53-24.csv
+ESS 400-1000,405,1.15
-ESS < 400,0,0,/srv/projects/party4d/archive/party2d_replication/outputs/diagnostics/convergence_summary_2026-06-12_12-53-24.csv
+ESS < 400,0,0
@@ -1,240 +0,0 @@
 # Data Coding Principles for 4D Latent Trait Model
 ## V4 Implementation (Current Version)
 **As of V4 (2025-11-18), manifesto items implement the bipolar bridge structure described in this document.**
 **Key Changes from V3.x**:
 - ✅ Manifesto items now load on TWO dimensions (bipolar bridges)
 - ✅ Data format: `type_high` and `type_low` columns replace single `type`
 - ✅ Stan model: Unified `Gamma_man` matrix replaces per-dimension arrays
 - ✅ Measurement consistency: Manifesto items match expert data structure
 **Why V4?**
 - Better identification (each observation informs two dimensions)
 - Estimated correlations (not imposed by construction)
 - No double-counting (each quasi-sentence counted once)
 See CHANGELOG.md for full V4 migration details.
 ---
 ## Model Structure Overview
 The model estimates **four unipolar latent dimensions**:
 - **pro_market**: Support for market liberalization
 - **pro_welfare**: Support for welfare state expansion
 - **cosmopolitan**: Support for internationalism, diversity, openness
 - **traditional**: Support for nationalism, security, traditional values
 These are **separate dimensions**, not two bipolar scales. Correlations between dimensions (e.g., cosmopolitan-traditional) are **estimated empirically**, not imposed by construction.
 ---
 ## Item Types and Loading Structure
 ### 1. Bipolar Bridge Items
 **Definition**: Items where the sample includes mentions of BOTH sides of an issue, and "positive" counts mentions favoring one pole.
 **Structure**:
 - `sample` = mentions of issue (any direction)
 - `positive` = mentions favoring one pole
 - `positive/sample` ratio varies from 0 to 1
 **Loading**: Should load on **ONE dimension only**
 **Examples**:
 **Manifesto Data**:
 ```
 var: "Multiculturalism"
 type: "cosmopolitan"
 sample: per607 (pro-multiculturalism) + per608 (anti-multiculturalism)
 positive: per607 (pro-multiculturalism)
 ```
 - High ratio → high cosmopolitan (party favors multiculturalism)
 - Low ratio → low cosmopolitan (party opposes multiculturalism)
 - Anti-multiculturalism is **implicitly measured** as (sample - positive)
 **PolDem Data**:
 ```
 var: "Immigration (Media)"
 type: "cosmopolitan"
 sample: all immigration mentions (direction != 0)
 positive: pro-immigration mentions (direction > 0)
 ```
 - High ratio → high cosmopolitan (media coverage shows party supporting immigration)
 - Low ratio → low cosmopolitan (media coverage shows party opposing immigration)
 ### 2. Why One Loading Suffices for Bipolar Items
 **Question**: Shouldn't anti-immigration also load on traditional?
 **Answer**: No, because:
 1. **Both poles are already captured**: The bipolar structure means low cosmopolitan (anti-immigration) is automatically measured
 2. **Avoids double-counting**: Each mention/quasi-sentence contributes to exactly ONE item
 3. **Empirical correlations emerge naturally**: If anti-immigration parties also score high on nationalism/law-and-order, the **posterior correlation** between cosmopolitan and traditional will reflect this
 4. **More flexible model**: Cosmopolitan-traditional relationship is **estimated**, not imposed
 **Imposed vs. Estimated Correlation**:
 - If we double-load immigration on both cosmopolitan (negative) and traditional (positive), we **force** them to be opposites
 - By loading only on cosmopolitan, we let the data reveal whether anti-immigration parties are also nationalist (empirical question)
 ---
 ## Coding Decision Rules
 ### Rule 1: Each Manifesto Code Appears in ONE Item Only
 **Good** (current structure):
 ```
 "Multiculturalism" (cosmopolitan):
  - per607 (Positive), per608 (Negative)
 "National Identity" (traditional):
  - per601 (Positive), per107 (Negative)
 ```
 - per607/per608 only in cosmopolitan
 - per601/per107 only in traditional
 - Correlation between dimensions is empirical
 **Bad** (double-loading):
 ```
 "Multiculturalism" (cosmopolitan):
  - per607 (Positive), per601 (Negative)
 "National Identity" (traditional):
  - per601 (Positive), per607 (Negative)
 ```
 - per601 and per607 counted twice
 - Imposes perfect negative correlation between cosmopolitan/traditional
 ### Rule 2: Stance Assignment Within Items
 Within each item (var), codes are assigned stance based on:
 - **Positive**: Codes indicating support for the item's construct
 - **Negative**: Codes indicating opposition to the item's construct
 **Example - "Internationalism" (cosmopolitan)**:
 - per107 (Internationalism positive): stance = "Positive"
 - per109 (Internationalism negative): stance = "Negative"
 - Result: High per107 / low per109 → high cosmopolitan score
 ### Rule 3: PolDem Direction Mapping
 PolDem uses `direction` variable (-1, 0, +1):
 - `direction > 0`: Support for the issue as coded
 - `direction < 0`: Opposition to the issue
 - `direction == 0`: Ambivalent (exclude from analysis)
 **Aggregation**:
 ```r
 poldem_processed %>%
  filter(direction != 0) %>%  # exclude neutral
  group_by(party, year, country, issue_cat) %>%
  summarise(
    sample = n(),  # all non-neutral mentions
    positive = sum(direction > 0)  # supportive mentions only
  )
 ```
 ---
 ## Special Cases
 ### Immigration (Direction Ambiguity)
 **Codebook says**: "Opposition to restrictive immigration"
 **Interpretation needed**: Does `direction = +1` mean:
 - A) Support for "opposition to restrictions" → pro-immigration (cosmopolitan)
 - B) Support for "restrictions" → anti-immigration (traditional)
 **Resolution**: Must manually inspect sample sentences before finalizing coding.
 If interpretation A is correct:
 ```r
 issue_cat == "immig" & direction > 0  → positive for cosmopolitan
 issue_cat == "immig" & direction < 0  → negative for cosmopolitan
 ```
 If interpretation B is correct:
 ```r
 issue_cat == "immig" & direction > 0  → negative for cosmopolitan
 issue_cat == "immig" & direction < 0  → positive for cosmopolitan
 # (REVERSED)
 ```
 ### Europe/Euro Items
 EU integration naturally maps to cosmopolitan-traditional dimension:
 **Manifesto Data**:
 - Add new items using per108 (EU integration positive) and per106 (EU integration negative)
 - Create separate vars: "EU Integration Support" (cosmopolitan), "Euroskepticism" (traditional)
 **PolDem Data**:
 ```r
 "EU Integration Support (Media)" (cosmopolitan):
  issue_cat = "europe" or "euro"
  sample = all mentions
  positive = direction > 0 (pro-EU)
 "Euroskepticism (Media)" (traditional):
  issue_cat = "europe" or "euro"
  sample = all mentions
  positive = direction < 0 (anti-EU)
 ```
 **Note**: Same sentences contribute to BOTH items, but counting opposite directions. This creates natural negative correlation between cosmopolitan/traditional.
 **Alternative approach** (cleaner, recommended): Load only on cosmopolitan:
 ```r
 "EU Position (Media)" (cosmopolitan):
  issue_cat = "europe" or "euro"
  sample = all mentions
  positive = direction > 0
 ```
 This is sufficient if we treat EU as a bipolar cosmopolitan item.
 ---
 ## Data Structure Requirements
 ### Manifesto Data Format (party-year-var level)
 Each row represents one item for one party-year:
 | party | country | year | var | type | sample | positive | project |
 |-------|---------|------|-----|------|--------|----------|---------|
 | 211 | DE | 2013 | Multiculturalism | cosmopolitan | 45 | 23 | Manifesto |
 | 211 | DE | 2013 | National Identity | traditional | 67 | 58 | Manifesto |
 | 211 | DE | 2013 | Economic Intervention | pro_welfare | 102 | 78 | Manifesto |
 - **var**: Item name (e.g., "Multiculturalism", "Economic Intervention")
 - **type**: Dimension it loads on (pro_market, pro_welfare, cosmopolitan, traditional)
 - **sample**: Total quasi-sentences mentioning this issue
 - **positive**: Quasi-sentences with positive stance toward this item
 ### PolDem Data Format (same structure)
 | party | country | year | var | type | sample | positive | project |
 |-------|---------|------|-----|------|--------|----------|---------|
 | 211 | DE | 2013 | Immigration (Media) | cosmopolitan | 23 | 8 | PolDem |
 | 211 | DE | 2013 | Nationalism (Media) | traditional | 15 | 12 | PolDem |
 Combined using `bind_rows()` to create unified dataset.
 ---
 ## Summary
 1. **Bipolar items load on one dimension only** - the ratio captures both poles
 2. **Each manifesto code appears in exactly one item** - no double-counting
 3. **Correlations between dimensions are estimated, not imposed** - more flexible model
 4. **Direction reversals are handled within items** - via stance assignment (Manifesto) or direction sign (PolDem)
 5. **All items must allow varying positive/sample ratios** - mix of positive and negative stances required
 This structure preserves the conceptual independence of the four dimensions while allowing the data to reveal their empirical relationships.
@@ -1,73 +0,0 @@
 # Full model run operations
 This note describes how to launch, monitor, and summarize the long full model
 run that generates the posterior estimates.
 Do not start a full run until raw-data and Stan-data preflight checks pass.
 ## Hardware note for runtime reporting
 The production run should record elapsed time for this workstation as:
 ```text
 Hardware: 4 cores of an AMD Ryzen 9 7945HX
 ```
 After the run, record the wall-time reported by `src/sh/show_run_progress.sh` or
 `outputs/model_outputs/latest/run_*/diagnostics/run_metrics.json`.
 ## Launch with durable logging
 From the repository root:
 ```bash
 mkdir -p outputs/logs
 ts="$(date +%Y%m%d_%H%M%S)"
 STAN_REFRESH=100 \
  nohup bash run_estimation.sh full \
  > "outputs/logs/full_run_${ts}.log" 2>&1 &
 echo $! > "outputs/logs/full_run_${ts}.pid"
 ```
 The wrapper configures local raw data, local R libraries, project-local temp
 space, and the writable project-local CmdStan copy by default.
 ## Monitor during the run
 Live log tail:
 ```bash
 tail -f outputs/logs/full_run_<timestamp>.log
 ```
 Convenience progress command:
 ```bash
 bash src/sh/show_run_progress.sh
 ```
 Stan progress is printed every `STAN_REFRESH` iterations (default `100`) and is
 captured in the durable log.
 ## Inspect after completion
 After a successful run, the chain CSVs are under:
 ```text
 outputs/model_outputs/latest/run_<timestamp>/chains/
 ```
 Run diagnostics and Stan logs are archived under:
 ```text
 outputs/model_outputs/latest/run_<timestamp>/diagnostics/
 ```
 Use:
 ```bash
 bash src/sh/show_run_progress.sh
 ```
 to print the latest run status, wall time, sampler configuration, divergences,
 tree-depth hits, and command-configuration verification status.
@@ -1,112 +0,0 @@
 # Raw data sources and local-only setup
 This repository includes model-ready inputs under `data/`, but it does not
 redistribute licensed, restricted, or third-party raw source files. Raw files
 needed to regenerate processed inputs should be kept in `_local/raw/` or another
 directory selected with `PARTY2D_RAW_DATA_DIR`.
 The source setup scripts document how to obtain and test those files locally.
 Users must use their own Manifesto Project API key and obtain the Morgan OCR/
 transcription file separately.
 The data setup workflow writes intermediates to `_local/build/`, regenerated
 inputs to `_local/generated-inputs/`, and comparison reports to `_local/reports/`.
 It does not overwrite committed `data/` files.
 Recommended local layout:
 ```text
 _local/raw/
  poldem/poldem-election_all.csv
  manifesto/MPDataset_MPDS2025a.csv
  partyfacts/partyfacts-external-parties.csv
  ches/...
  vparty/...
  poppa/...
  gps/...
  morgan/...
 ```
 The scripts read `PARTY2D_RAW_DATA_DIR` when it is set. For another location, use:
 ```bash
 export PARTY2D_RAW_DATA_DIR=/path/to/local/raw
 ```
 ## Required raw inputs
 | Source | Raw file | Local path below `$PARTY2D_RAW_DATA_DIR` | Used by | Redistribution |
 | --- | --- | --- | --- | --- |
 | Manifesto Project Dataset | `MPDataset_MPDS2025a.csv` | `manifesto/MPDataset_MPDS2025a.csv` | `data-setup/R/process_manifesto.R` | Not redistributed in this repo |
 | PolDem Election Campaigns, all issues | `poldem-election_all.csv` | `poldem/poldem-election_all.csv` | `data-setup/R/process_poldem.R` | Not redistributed in this repo |
 | CHES family | CHES aggregate and expert-level files | `ches/` | `data-setup/R/process_expert.R` | Not redistributed in this repo |
 | V-Party | `V-Dem-CPD-Party-V2.rds` | `vparty/V-Dem-CPD-Party-V2.rds` | `data-setup/R/process_expert.R` | Not redistributed in this repo |
 | POPPA | `poppa_integrated_v2.rds` | `poppa/poppa_integrated_v2.rds` | `data-setup/R/process_expert.R` | Not redistributed in this repo |
 | Global Party Survey 2019 | `Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab` | `gps/Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab` | `data-setup/R/process_expert.R` | Not redistributed in this repo |
 | Morgan historical expert data | `morgan_positions_raw.csv` | `morgan/morgan_positions_raw.csv` | `data-setup/R/process_morgan.R` | Not redistributed in this repo |
 | PartyFacts crosswalk | `partyfacts-external-parties.csv` | `partyfacts/partyfacts-external-parties.csv` | source harmonization scripts | Not redistributed in this repo |
 ## PolDem download
 Dataset: `poldem-election_all` from the PolDem Election Campaigns collection.
 - Overview: <https://poldem.eui.eu/data-overview/>
 - Download page: <https://poldem.eui.eu/download/election-campaigns/>
 - CSV URL used locally on 2026-06-11:
  <https://poldem.eui.eu/downloads/cosa/poldem-election_all.csv>
 Observed checksum after download on 2026-06-11:
 ```text
 sha256  2cd8c9108b1b0b9c1b6594bb21acee709c70259cd02f450bc69fc09b505fc9fb
 ```
 ## Local source check and rebuild
 To check local raw file placement and print byte sizes/checksums directly, run:
 ```bash
 bash data-setup/check_raw_data.sh
 ```
 Run the full source-data setup, rebuild, and comparison test with:
 ```bash
 bash data-setup/run_data_setup.sh
 ```
 The generated files remain under `_local/generated-inputs/`. They are compared to
 the committed model-ready inputs, but never copied over them automatically.
 ## Processed files kept in this repository
 The committed files under `data/` are limited to the model-ready inputs used by
 the Julia/Stan estimation path:
 - `data/text_data.csv`
 - `data/expert.csv`
 - `data/lr_data.csv`
 - `data/union_mapping.csv`
 - `data/party_families.csv`
 These files document the analysis-ready inputs while avoiding redistribution of
 the underlying raw source data.
 See `data-setup/source_manifest.csv` for a machine-readable source checklist.
 ## Scripted download status
 `data-setup/R/01_download_sources.R` downloads sources that are script-accessible
 under provider terms: PolDem, PartyFacts, CHES family files where provider links
 are live, POPPA from Harvard Dataverse, GPS from Harvard Dataverse, Manifesto
 with user credentials, and V-Party through the provider form when
 `PARTY2D_VDEM_EMAIL` is set.
 The Manifesto Project main dataset requires Manifesto Project access/API
 credentials from the provider. Set `MANIFESTO_API_KEY` or
 `PARTY2D_MANIFESTO_API_KEY` for scripted download.
 The Morgan historical file is a local OCR/transcription source from Morgan
 (1976), not a public provider download. It can be provided on request and must be
 placed locally at `$PARTY2D_RAW_DATA_DIR/morgan/morgan_positions_raw.csv` for
 full rebuild tests.
@@ -1,350 +0,0 @@
 # Party Union Mapping
 ## Individual Party Guarantee
 **Every `party_id` in the output CSV represents an individual political party, never an electoral alliance or bloc.**
 Electoral alliances and blocs are handled in one of two ways:
 1. **Decomposed via mean-constituent averaging** (N=123 mappings): Shared manifesto data feeds into individual constituent party estimates. The output contains the constituents, not the alliance.
 2. **Excluded with documented justification** (see "Excluded Alliance Labels" below): Alliance labels with no mappable constituents are dropped from the output.
 Post-estimation verification in `02_post_estimation.jl` hard-fails if any union/alliance PF ID appears in the output.
 ## Overview
 Expert surveys often rate individual constituent parties separately, while manifesto data is published under a union/coalition/merged party name. This document maps individual party PartyFacts IDs to the union PartyFacts IDs used in `text_data.csv`.
 **Direction:** Individual party (expert data) → Union (manifesto/text data)
 ## Model Integration (V4 Mean-Constituent Model)
 The V4 Stan model (`stan_model_2dim_v4.stan`) uses these mappings to produce **individual party estimates** for all constituent parties. Instead of collapsing expert data onto the union ID, V4 gives every constituent its own latent position (theta) and links shared manifesto data to the **mean** of constituent thetas.
 ### How it works
 1. **Each constituent gets its own segment and random walk.** CDU (1375) and CSU (1731) each have independent theta trajectories. The union ID (211) gets NO segment.
 2. **Manifesto observations constrain the mean.** A CDU/CSU manifesto in year t enters the likelihood as:
   ```
   pos = mean(theta[dim, rr_CDU_t], theta[dim, rr_CSU_t])
   ```
   The observation is counted once (no double-counting), but it pulls both constituents' thetas via their average.
 3. **Expert data constrains individuals directly.** A CHES rating of CDU in 2019 maps to `theta[dim, rr_CDU_2019]` with no averaging. This is what identifies the *difference* between constituents.
 4. **Identification depends on data availability:**
   - **Periods with individual expert data** (e.g., CHES 1999-2024): Constituent estimates separate meaningfully. CSU appears more traditionalist on the cultural dimension than CDU, matching known ground truth.
   - **Periods without individual expert data** (e.g., 1950s-1990s): Only the shared manifesto constrains the mean. The random walk prior pulls constituents toward similar values, so CDU ≈ CSU with wide credible intervals on the gap. The estimates gradually differentiate as expert data appears.
 5. **Backwards compatible.** With an empty `union_mapping.csv`, all observations have `n_const=1` and V4 reduces exactly to V3.
 ### Data flow
 ```
 Data setup pipeline (`data-setup/R/02_build_model_inputs.R`):
  - text_data keeps union PF IDs (211) for manifesto rows
  - expert/lr_data keeps individual PF IDs (1375, 1731)
  - Expert filtering: party in text_data OR party in constituent_parties
 Julia pipeline:
  02_data_loading.jl  → loads union_mapping.csv → builds union_to_constituents dict
  03_data_preparation.jl → creates segments for CONSTITUENTS (not unions)
                         → builds flat arrays: n_const_man[], const_offset_man[], const_rr_man[]
  04_model_execution.jl  → passes constituent arrays to Stan
 Stan model (stan_model_2dim_v4.stan):
  - Manifesto likelihood: averages theta over constituents per observation
  - Expert likelihoods: same averaging (nc=1 for individual obs = direct lookup)
  - No new parameters vs V3 — only data block and likelihood computation change
 ```
 ### Output
 Post-estimation produces individual party rows (CDU=1375 and CSU=1731 as separate rows) with a `union_party_id` column (= 211 for both, NA for standalone parties). The anchor party is CDU (1375) instead of CDU/CSU (211).
 ### Scale impact (smoke test, post-1990 data)
 - 471 parties → 483 segments, 461 with valid segments (vs ~449 without unions)
 - 708 union manifesto observations with multi-constituent averaging (3.2% of 21,991 total)
 - 23,427 total constituent entries in flat arrays
 - 9,870 segment-year positions (R)
 - No new model parameters (theta simply covers more segment-years)
 - Negligible performance impact: `nc=1` fast path for >96% of observations
 ### Data quality (verified 2026-02-08)
 - **No double-counting**: No constituents appear in text_data alongside their union
 - **Union IDs verified absent from output** via post-estimation check in `02_post_estimation.jl`
 - **No chain mappings**: No PF ID serves as both union target and constituent of another union
 - **No duplicate rows** in text_data after deduplication
 - **All flagged-for-review parties** in audit script are confirmed individual parties (false positives from name patterns)
 ## Detection Methodology
 `scripts/diagnose_party_mismatches.R` uses a multi-signal approach, ranked by reliability:
 ### Phase 1: MARPOR `progtype` variable (definitive)
 The raw Manifesto Project data (`MPDataset_MPDS2025a.csv`) contains a `progtype` variable classifying each manifesto entry:
 | progtype | Meaning | Relevance |
 |----------|---------|-----------|
 | 1 | Party's own manifesto | Individual party, no union issue |
 | **2** | **Programme of 2+ parties (individual tracking)** | Joint manifesto: each constituent gets its own CMP code with identical scores |
 | 3 | Electoral manifesto by a single party | Individual party |
 | **4** | **Estimated from another party's programme** | Party had no manifesto, inherited another party's scores |
 | **5** | **Average of member parties' manifestos** | MARPOR computed average from constituent parties |
 | 6 | Other | Miscellaneous |
 | **8** | **Party bloc programme** | Bloc-level CMP code representing multiple parties |
 | 9 | Non-standard text | Various sources |
 **Phase 1a (progtype=2):** Group by `(country, date)` where progtype=2 to find which CMP codes shared a joint manifesto at each election. Map each to PF IDs. If some are in text_data and others are expert-only, create constituent mappings.
 **Phase 1b (progtype=5):** Average-of-members entries have their own CMP code. Find constituents via PartyFacts "composed of:" comments and manifesto party names.
 **Phase 1c (progtype=8):** Bloc-level entries have their own CMP code. Find constituents via PartyFacts "composed of:" comments and manifesto party names.
 **Phase 1d (progtype=4):** Proxy entries where MARPOR estimated from another party. Currently detects 0 expert-only parties with progtype=4.
 ### Phase 2: PartyFacts metadata enrichment
 **Phase 2a (comment parsing):** Search text_data party comments for "composed of:" patterns. Match mentioned abbreviations against expert-only parties using word-boundary matching.
 **Phase 2b (parlgov "+" notation):** parlgov uses "+" to denote unions (e.g., "CDU+CSU", "CCD+CDU"). Parse fragments and match against expert-only parties.
 **Phase 2c (name fragment matching):** For text_data parties with "/" or "-" in names, split into fragments and match against expert-only parties. **Bilingual disambiguation:** if all fragments resolve to the same PF ID, the "/" separates language variants (e.g., Swiss SPS/PSS), not constituents.
 ### Phase 3: LLM verification (optional)
 For remaining unmatched expert-only parties, uses gpt-4.1 via GESIS OpenWebUI with enriched prompts containing: all name_short values across datasets, year ranges, PartyFacts comments, progtype history, and the full list of text_data parties in the same country. Skip with `--skip-llm` flag.
 ### Matching safeguards
 - **Word-boundary matching:** Uses `\b` regex boundaries to prevent substring false positives (e.g., "DS" matching "NDSI")
 - **Minimum 2-character terms:** Single-letter abbreviations (e.g., "K", "G") are excluded from matching
 - **Deduplication:** When the same mapping is detected by multiple methods, the highest-reliability method is kept (manual > progtype > comments > parlgov > name_fragment > LLM)
 - **Idempotent:** The script strips `detection_method` from existing mappings on re-run
 ## How This Was Built
 1. **Original 36 mappings** (2026-02-06/07): String matching + LLM + RA verification + independent research
 2. **Expanded with progtype** (2026-02-07): Rewrote `scripts/diagnose_party_mismatches.R` to use MARPOR progtype (types 2, 4, 5, 8), PartyFacts comments, parlgov "+", and bilingual-aware name matching
 3. **LLM verification with gpt-4.1** (2026-02-07): Ran enriched prompts through gpt-4.1 via GESIS OpenWebUI for 345 remaining expert-only parties. Found 32 additional verified constituent mappings across 25 countries.
 4. **Bloc-centric LLM sweep** (2026-02-07): For each of 38 unmapped progtype=8 bloc parties in text_data, sent targeted gpt-4.1 queries asking which expert-only parties are actual constituents. Found 17 new mappings (Brazil PT-led coalitions, Chilean Concertación, Serbian blocs, Latvian unions, etc.).
 5. **Manual research** (2026-02-07): Verified remaining unmapped blocs. Added 4 manual mappings (PL: .N→KO; CL: PC→Concertación, PH→Concertación; ES: Amaiur→EH Bildu). Confirmed remaining 27 blocs have no mappable expert-only constituents (constituents either already in text_data, not in expert surveys, or in fluid coalitions).
 6. **Temporary election coalitions** (2026-02-08): Systematic analysis of progtype=2 joint manifesto groups. Grouped by identical CMP content (all 142 per* columns) to separate left and right coalitions within the same election. Found 9 unmapped expert-only parties that shared manifestos with text_data parties (Italy 2001/2006/2013, France 2017). Added CCD→CdL from orphan analysis (1 manual). Total: 10 new mappings.
 7. **Individual party guarantee cleanup** (2026-02-08): Removed 11 progtype_2/progtype_2_joint entries that mapped real individual parties as unions of zero-data coalition partners. Removed 2 chain mappings (union→union→constituent). Added 4 new mappings for genuine alliances (MX: Salvemos a México→{PRI, PVEM}; IL: Joint List→{Hadash, Balad}). Documented classification decisions for all 15 dual-progtype parties and 4 pure bloc labels. Added post-estimation verification check and audit script.
 ## Mapping Table
 The canonical mapping is in `data/union_mapping.csv` with columns:
 - `manifesto_pf_id` — PartyFacts ID of union (in text_data)
 - `manifesto_name` — Union display name
 - `expert_pf_id` — PartyFacts ID of individual constituent (in expert_raw)
 - `expert_name` — Individual party display name
 - `country` — ISO2 country code
 - `relationship` — Description of the relationship
 - `status` — `implemented` (active in pipeline) or `pending` (awaiting implementation)
 - `detection_method` — How detected: `manual`, `progtype_2`, `progtype_2_joint`, `progtype_5`, `progtype_8`, `progtype_4`, `composed_comment`, `parlgov_plus`, `name_fragment`, `llm_verified`
 ## Detection Method Breakdown
 | Method | Count | Description |
 |--------|-------|-------------|
 | llm_verified | 48 | gpt-4.1 verified (party-centric + bloc-centric) |
 | manual | 45 | Hand-verified mappings (includes MX/IL bloc mappings, CCD→CdL) |
 | progtype_8 | 23 | Bloc-level CMP entries matched to expert-only constituents |
 | name_fragment | 4 | "/" or "-" name splitting matched to expert-only parties |
 | composed_comment | 2 | PartyFacts "composed of:" comments |
 | progtype_5 | 1 | Average-of-members entries (Croatia ZL) |
 **Total: 123 rows** mapping 116 unique expert parties to 87 union parties across 41 countries
 ## Bloc Coverage
 Of 49 progtype=8 bloc parties in text_data, 22 (44.9%) have at least one constituent mapped. The remaining 27 have no mappable expert-only constituents because:
 - Actual constituents already have their own text_data entries (e.g., KDNP, Yesh Atid, TB-LNNK)
 - Actual constituents are not in expert surveys (e.g., Hadash, Ra'am, Balad, VS, DKP)
 - Coalition membership is too fluid for static mapping (Panama, Colombia, Israel shifting coalitions)
 - The entry is a single party coded as a bloc (e.g., German Minority, Red-Green Unity List)
 ## Temporary Election Coalitions (progtype=2) — REMOVED
 The progtype=2 joint manifesto mappings (Italy 2001/2006/2013, France 2017, Belgium 1971) were removed in the 2026-02-08 cleanup because they mapped real individual parties as "unions" of tiny coalition partners with zero data. See "Removed Mappings" below for details.
 The CCD(1767)→CdL(6241) mapping from orphan analysis remains, as CCD was a genuine long-term CdL constituent (progtype=1 in 1996, part of CdL bloc from 2001).
 ## Unmappable Expert-Only Parties
 These parties appear in expert surveys but have no manifesto union to map to. Verified against `text_data_unfiltered.csv` (pre-temporal-filter, 1238 parties) on 2026-02-07.
 **Filter retest results:** 54 of the 80 original RA task parties have manifesto data in the unfiltered text_data, but all were dropped by the temporal continuity filter (requires ≥3 years with gaps ≤6). These parties have their own CMP data but too few observations. They are correctly handled via union mapping (where applicable) or excluded (where standalone). See `scripts/party_mismatch_ra_task.csv` for full year-level detail.
 | Expert PF ID | Name | Country | Manifesto years (unfiltered) | Expert years | Why unmappable |
 |-------------|------|---------|------------------------------|--------------|----------------|
 | 5623 | Compromís | ES | 0 (CMP codes map to different PF IDs) | 2 (2018, 2023) | No manifesto data under PF ID 5623. CMP codes 33098/33093/33914 are separate PartyFacts entries. |
 | 4363 | FDG (Front de Gauche) | FR | 1 (2012) | 1 (2014) | 1 CMP year only. PCF (1251) already has its own expert data. |
 | 5731 | NNP | ZA | 0 | 1 (1999) | Apartheid party that dissolved into ANC 2005. Ideologically incompatible mapping. |
 | 5553 | FREPASO | AR | 1 (1995) | 4 (1995-2001) | 1 CMP year only. Independent party; Alianza (1999) was separate CMP entity. |
 | 8122 | CF (Consenso Federal) | AR | 1 (2019) | 2 (2019-2020) | 1 CMP year only. One-off coalition. |
 | 4182 | FPL+UCeDé | AR | 1 (2003) | 3 (1987-1991) | 1 CMP year only. Incoherent entity (UCeDé 1987 ≠ FPL coalition 2003). |
 | 6160 | FR (Frente Renovador) | AR | 0 | 4 (2013-2019) | No CMP code at all. Major ideological shift 2013→2019. |
 | 5879 | CD (Centro Democrático) | CO | 1 (2014) | 3 (2014-2019) | 1 CMP year only. Uribe's party, independent. |
 | 4411 | PNI/PPN | CR | 0 | 1 (1974) | No CMP code. Defunct 1970s party. |
 | 7412 | EK/DEK | CY | 0 | 1 (1970) | No CMP code. Defunct 1970 far-right party. |
 | 3935 | United Opposition | GE | 1 (2008) | 1 (2008) | 1 CMP year only. Was anti-UNM alliance (not led by UNM). |
 ## Rejected Mappings
 These were proposed by the RA but found incorrect during independent research:
 | Individual PF ID | Name | Proposed target | Why wrong |
 |-----------------|------|-----------------|-----------|
 | 5623 | CC/Compromís | 81 (CCa-PNC-NC) | Name coincidence. Compromís is a Valencian left party; CCa is a Canarian right party. |
 | 4363 | FDG | 1251 (PCF) | FDG was a multi-party alliance (PCF + Parti de Gauche). PCF already has its own 2014 expert data. Mapping FDG→PCF would double-count. |
 | 5731 | NNP | 1219 (ANC) | The NNP (ex-apartheid National Party) dissolved into ANC in 2005 as political capitulation. NNP positions are diametrically opposed to ANC on every dimension. |
 ## Completeness Audit (2026-02-08)
 Systematic analysis of 631 orphan expert parties (with 4+ observations) across 46 countries (excluding FPTP systems like UK, US, Canada where party unions are not a meaningful concept). Cross-referenced with MARPOR progtype data and PartyFacts metadata.
 ### Methodology
 1. **Progtype=2 sweep**: Identified all 93 MARPOR entries with progtype=2. Grouped into 27 content-identical subgroups. Found 12 expert-only parties with progtype=2 data; 10 were mappable (mapped above), 2 had no matching text_data party in their coalition.
 2. **Orphan analysis**: For each of 46 countries, identified expert-only parties not in text_data and not already mapped. Examined PartyFacts metadata (names, comments, ideology tags) and MARPOR progtype history for each.
 3. **Coalition verification**: For Italian orphans specifically, checked each party's MARPOR entries for progtype=2/8 membership and ideological alignment with existing text_data coalition parties.
 ### Major standalone orphan parties (not coalition members)
 These are significant parties with substantial expert data but no manifesto/union data. They are genuinely standalone — not missed union constituents.
 | Country | Party | PF ID | Expert obs | Why standalone |
 |---------|-------|-------|-----------|----------------|
 | AT | BZÖ | 599 | 7 | Splinter from FPÖ; own CMP data filtered out |
 | BE | PTB/PVDA | 1753 | 14 | Independent far-left; never part of any coalition |
 | CZ | ANO | 2141 | 13 | Babiš party; own CMP data exists but too recent |
 | CZ | Piráti | 2047 | 10 | Standalone; own CMP data exists |
 | FR | REM/R | 5857 | 10 | Macron's party; too new for sufficient CMP data |
 | FR | FI | 5858 | 8 | Mélenchon's party; standalone |
 | IT | M5S | 2046 | 13 | Five Star Movement; progtype=1 only, standalone |
 | IT | FDI | 2280 | 10 | Fratelli d'Italia; progtype=1 only, standalone |
 | RO | USD | 120 | 23 | Social democratic bloc; own CMP code exists |
 | SK | OĽaNO | 2130 | 13 | Populist party; standalone |
 | CO | PCC | 1577 | 19 | Conservative party; own CMP code exists but filtered |
 | BR | PSDB | 225 | 13 | Social democrats; standalone (not PT coalition) |
 ### Inactive union mappings
 These union targets exist in `union_mapping.csv` but are NOT in text_data, making their constituent mappings inactive:
 | Union PF ID | Name | Country | Constituents | Why inactive |
 |-------------|------|---------|-------------|--------------|
 | 1212 | SEL | IT | SEL(7031) | 1212 not in text_data |
 | 1737 | Olive Tree | IT | PCI(34), DS(878) | 1737 not in text_data; only 2 MARPOR years |
 Note: 962 (CCD+CDU→CDU-Italy) was removed from the mapping entirely because it created a chain (962 is both a constituent of UdC and a union target). See "Removed Mappings" below.
 ### Conclusion
 The 123 mappings comprehensively cover: (1) all permanent unions with separate expert data, (2) all progtype=8 bloc parties with mappable expert-only constituents, and (3) CCD as an additional Italian coalition member identified through orphan analysis. Remaining orphan expert parties are genuinely standalone parties whose manifesto data was either filtered out by temporal continuity requirements or does not exist.
 ## Removed Mappings (2026-02-08)
 ### 11 progtype=2 joint manifesto entries removed
 These entries mapped real individual parties (with substantial text and expert data) as "unions" of temporary coalition partners that had zero text data AND zero expert data. Every constituent had no information to contribute, making the mapping harmful (it reduced real parties to averages with phantom partners).
 | Union PF ID | Union Name | Constituent PF ID | Constituent | Country | Why removed |
 |---|---|---|---|---|---|
 | 8054 | DS | 279 | M-DL | IT | M-DL has 0 text, 0 expert data; DS has 120 text, 3 expert |
 | 1404 | PRC | 1635 | PdCI | IT | PdCI has 0 text, 0 expert data; PRC has 58 text, 15 expert |
 | 1404 | PRC | 1711 | RnP | IT | RnP has 0 text, 0 expert data |
 | 6241 | CdL | 888 | NPSI | IT | NPSI has 0 text, 0 expert data; CdL's progtype_8 mappings (AN, CeD, FI) remain |
 | 6241 | CdL | 2415 | ALD | IT | ALD has 0 text, 0 expert data |
 | 768 | IdV | 115 | P-UDEUR | IT | P-UDEUR has 0 text, 0 expert data; IdV has 26 text, 8 expert |
 | 768 | IdV | 1369 | SVP | IT | SVP has 0 text, 0 expert data (under this PF ID) |
 | 1221 | Lega | 365 | PdL | IT | PdL has 0 text, 0 expert data; Lega has 77 text, 24 expert |
 | 1595 | UMP | 3229 | UDI | FR | UDI has 0 text, 0 expert data; UMP has 66 text, 18 expert |
 | 49 | openVLD | 622 | CD&V | BE | CD&V is a large party (own PF ID); mapping created false dependency |
 | 554 | PRL | 622 | CD&V | BE | Same issue: CD&V already has its own data pipeline |
 ### 2 chain mapping entries removed
 These created chain dependencies (union A → union B → constituents), which the pipeline does not support:
 | Union PF ID | Union Name | Constituent PF ID | Constituent | Country | Why removed |
 |---|---|---|---|---|---|
 | 962 | CCD+CDU | 763 | CDU (Italy) | IT | 962 is itself a constituent of UdC (201). CDU-Italy (763) has 0 data. Chain: 201→962→763. |
 | 5939 | PàF | 1742 | AD | PT | 5939 is itself a constituent of CDS-PP (1308). AD (1742) has 0 data. Chain: 1308→5939→1742. |
 ## New Mappings Added (2026-02-08)
 | Union PF ID | Union Name | Constituent PF ID | Constituent | Country | Rationale |
 |---|---|---|---|---|---|
 | 3979 | Salvemos a México | 1474 | PRI | MX | PRI-PVEM electoral coalition (2006-2012); PRI has extensive text + expert data |
 | 3979 | Salvemos a México | 446 | PVEM | MX | PVEM is second constituent; has its own text + expert data |
 | 7912 | Joint List | 421 | Hadash | IL | Arab party coalition (2015-2021); Hadash has CHES 2022 data |
 | 7912 | Joint List | 1663 | Balad | IL | Balad is constituent; has CHES 2022 data |
 ## Excluded Alliance Labels
 These party IDs appear in text_data as bloc/alliance labels but are NOT decomposed via union mapping because no constituent has data in the pipeline:
 | PF ID | Name | Country | Text data | Expert data | Why excluded |
 |---|---|---|---|---|---|
 | 3995 | Alianza Acción Opositora | PA | 48 obs | 0 | No expert survey coverage for Panama. No constituents identifiable in pipeline. |
 Note: Several other progtype=8 bloc parties in text_data also have no mapped constituents (see "Bloc Coverage" below), but they remain in the output either because (a) they have their own expert data (e.g., 2988 Georgian Dream, 3916 Alianza Grande) or (b) they function as individual parties despite bloc coding (e.g., 1527 Enhedslisten, 1439 German Minority).
 ## Classification Decisions
 ### Dual-progtype parties (both progtype=1 and progtype=8)
 These 15 parties have MARPOR entries under both individual (progtype=1/3) and bloc (progtype=8) codes. Each was individually reviewed.
 **Classified as individual parties (no action needed):**
 | PF ID | Name | Country | Evidence |
 |---|---|---|---|
 | 57 | SLD | PL | Dominant Polish left party; bloc coding reflects coalition leadership, not alliance status |
 | 81 | CCa | ES | Canarian regionalist party; single party with local coalition leadership |
 | 1056 | SC | LV | Latvian party; dual coding reflects different election formats |
 | 1150 | SDE | EE | Estonian Social Democrats; individual party |
 | 1396 | Samfylkingin | IS | Icelandic Social Democratic Alliance; merged into single party |
 | 1439 | MN | PL | German Minority in Poland; single ethnic party coded as bloc |
 | 1527 | Enhedslisten | DK | Red-Green Alliance; functions as single party since 1989 |
 | 1691 | FiDeSz-KDNP | HU | FiDeSz dominant; KDNP (1412) has separate PF ID and data |
 | 2172 | ENM | GE | United National Movement; single party with bloc-era coding |
 | 2228 | BYuT | UA | Tymoshenko bloc; functions as single Ukrainian party |
 | 2252 | Yabloko | RU | Russian liberal party; individual entity |
 **Classified as individual parties after research:**
 | PF ID | Name | Country | Decision | Evidence |
 |---|---|---|---|---|
 | 506 | VL-TB/LNNK | LV | Individual (merger party) | National Alliance formed 2010 by merger of VL and TB/LNNK. Post-merger, functions as single party. MARPOR data 2010-2022. TB/LNNK (1704) has separate pre-merger data (1998-2014). Not mapped as union because 1704 is already a union target with its own constituents; mapping would create a chain. |
 | 1586 | sp.a-SPIRIT | BE | Already handled | Already mapped as constituent of sp.a (1680) in union_mapping.csv. 0 text, 0 expert data under this PF ID. |
 | 7599 | Kahol Lavan | IL | Individual party | Short-lived centrist party (2019-2020). Has own expert data (CHES 2021, V-Party 2019). Unified entity, not a multi-party alliance. |
 | 2988 | Georgian Dream | GE | Individual party | Despite progtype=8 coding, functions as a single party-movement. Has own expert data (GPS 2019, V-Party 2012/2016). |
 | 3916 | Alianza Grande | CO | Individual party (catch-all PF ID) | PF ID covers multiple Colombian coalitions. Has own expert data (CHES 2020). No separate constituent expert data exists. |
 **Classified as alliance and mapped:**
 | PF ID | Name | Country | Decision | Evidence |
 |---|---|---|---|---|
 | 7912 | Joint List | IL | Alliance → mapped to Hadash (421), Balad (1663) | Arab party coalition (2015-2021). GPS explicitly names 4 constituents. Hadash and Balad have separate CHES 2022 data and their own text_data. |
 | 3979 | Salvemos a México | MX | Alliance → mapped to PRI (1474), PVEM (446) | PRI-PVEM electoral coalition. Both constituents have extensive text and expert data. |
 ## Audit Methodology
 The union-mapping audit checks every party in the output CSV:
 1. **Union mapping check**: Verifies no `manifesto_pf_id` from `union_mapping.csv` appears in output (hard fail).
 2. **Constituent check**: Identifies parties that are `expert_pf_id` in the mapping (expected: these are individual constituents of unions).
 3. **Expert data check**: Flags parties with no expert survey data (text-only entities).
 4. **Name pattern check**: Scans PartyFacts names for alliance indicators (keywords: alliance, coalition, bloc, front, union, alianza, frente; characters: +, /, &).
 5. **Classification**: Each party gets one of: `individual_party`, `flagged_for_review`, `error_union_in_output`.
 **Post-estimation verification** (`02_post_estimation.jl`): After extracting estimates, loads all `manifesto_pf_id` values from `union_mapping.csv` and checks none appear in the output `party_id` column. If any do, the script errors with a hard fail.
@@ -918,7 +918,7 @@ end
 function main()
    println("="^60)
-    println("POST-ESTIMATION: 4D Latent Trait Model (V10)")
+    println("POST-ESTIMATION: Party-position model")
    println("="^60)
    println("Started: $(Dates.format(now(), "yyyy-mm-dd HH:MM:SS"))")
@@ -95,7 +95,7 @@ const EXPERT_VAR_TO_DIM = Dict(
 )
 function load_and_preprocess_4dim_data(start_year=1950; data_dir::String=".")
-    println("Loading 4D latent trait data files...")
+    println("Loading party-position data files...")
    println("Start year filter: $start_year")
    data_dir != "." && println("Data directory: $data_dir")
@@ -274,4 +274,4 @@ end
 if abspath(PROGRAM_FILE) == @__FILE__
    text_data, expert_dim, expert_lr, year0, u2c, c2u = load_and_preprocess_4dim_data()
    println("4D data loading test completed successfully")
-end
+end
@@ -1,7 +1,7 @@
 #!/usr/bin/env julia
 #############################################################################
 ## 04_model_execution_4dim.jl
-## Stan model compilation and execution for 4D latent trait model
+## Stan model compilation and execution for the party-position model
 ## Based on v9 execution but adapted for four dimensions
 #############################################################################
@@ -601,7 +601,7 @@ function create_4dim_init_function(dat_4dim, J, P, R, T_year, N_ciy; model_versi
        # SOLUTION: Use explicit Vector{Vector} to guarantee correct JSON structure
        # V10: theta_init_raw has S rows (segments), not J rows (parties)
        base_init = Dict{String, Any}(
-            # 4D latent trait parameters - Vector of Vectors for correct JSON
+            # Four-trait legacy initialization branch - Vector of Vectors for correct JSON
            "theta_ncp" => [zeros(R) for _ in 1:4],              # 4 rows of R elements
            "theta_init_raw" => [zeros(S) for _ in 1:4],         # 4 rows of S elements (V10: segments)
            "sigma_theta_init" => ones(4),           # SD per dimension
@@ -2,14 +2,14 @@
 #############################################################################
 ## 05_results_processing.jl
 ## Extract and process 4D model results with diagnostics
-## Adapted from old_project for latent traits only (no election effects)
+## Extract and process model results without election effects
 #############################################################################
 using StanSample, DataFrames, Statistics
 function extract_model_results_4dim(stanmodel)
    """
-    Extract model results for 4D latent trait model
+    Extract model results for the party-position model
    Simplified version - no election effects (pure latent traits)
    """
    println("Extracting 4D model results...")
@@ -17,7 +17,7 @@ function extract_model_results_4dim(stanmodel)
    try
        println("Model completed successfully - extracting results")
-        # For 4D latent trait model, we save the full stanmodel object
+        # Save the full stanmodel object for downstream processing
        # Post-estimation will extract specific parameters later
        return (
@@ -294,7 +294,7 @@ function generate_readme(
    open(filepath, "w") do f
        write(f, "=" ^ 78 * "\n")
-        write(f, "4D LATENT TRAIT MODEL - MODEL RUN RESULTS\n")
+        write(f, "PARTY-POSITION MODEL - MODEL RUN RESULTS\n")
        write(f, "=" ^ 78 * "\n\n")
        write(f, "Run ID: $run_id\n")
@@ -1,311 +0,0 @@
 # ============================================================
 # 00_data-management.R - Master Data Pipeline Orchestrator
 # ============================================================
 # Coordinates all data processing sub-scripts and produces
 # final output files for the 4D latent trait model.
 #
 # Sub-scripts (run conditionally based on intermediate file existence):
 #   00a_process_manifesto.R  -> manifesto_data.csv
 #   00c_process_poldem.R     -> poldem_data.csv
 #   00d_process_expert.R     -> expert_raw.csv, lr_data_raw.csv
 #   00e_process_morgan.R     -> morgan_data.csv, morgan_lr.csv
 #
 # Final outputs:
 #   text_data.csv - Combined manifesto + PolDem
 #   expert.csv    - Expert survey data (CHES, V-Party, POPPA, GPS)
 #   lr_data.csv   - General left-right anchoring data
 # ============================================================
 library(tidyverse)
 library(countrycode)
 # Set working directory (works both in RStudio and command line)
 if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
 }
 cat("============================================================\n")
 cat("Data Management Pipeline\n")
 cat("============================================================\n\n")
 # ============================================================
 # Configuration: Set to TRUE to force re-run of sub-scripts
 # ============================================================
 FORCE_RERUN_MANIFESTO <- FALSE
 FORCE_RERUN_POLDEM <- FALSE
 FORCE_RERUN_EXPERT <- FALSE
 FORCE_RERUN_MORGAN <- FALSE
 # ============================================================
 # Step 1: Manifesto Data
 # ============================================================
 cat("Step 1: Manifesto data\n")
 if (!file.exists("manifesto_data.csv") || !file.exists("election_data.csv") || FORCE_RERUN_MANIFESTO) {
  cat("  Running 00a_process_manifesto.R...\n")
  source("../src/r/00a_process_manifesto.R")
 } else {
  cat("  Loading cached manifesto_data.csv and election_data.csv...\n")
 }
 manifesto <- read_csv("manifesto_data.csv", show_col_types = FALSE)
 election_data <- read_csv("election_data.csv", show_col_types = FALSE)
 cat(sprintf("  Loaded manifesto: %d rows, %d parties\n", nrow(manifesto), n_distinct(manifesto$party)))
 cat(sprintf("  Loaded election: %d rows, %d parties\n\n", nrow(election_data), n_distinct(election_data$party)))
 # ============================================================
 # Step 2: PolDem Media Data
 # ============================================================
 cat("Step 2: PolDem media data\n")
 if (!file.exists("poldem_data.csv") || FORCE_RERUN_POLDEM) {
  cat("  Running 00c_process_poldem.R...\n")
  source("../src/r/00c_process_poldem.R")
 } else {
  cat("  Loading cached poldem_data.csv...\n")
 }
 poldem_data <- read_csv("poldem_data.csv", show_col_types = FALSE)
 cat(sprintf("  Loaded: %d rows, %d parties\n\n", nrow(poldem_data), n_distinct(poldem_data$party)))
 # ============================================================
 # Step 4: Expert Survey Data
 # ============================================================
 cat("Step 3: Expert survey data\n")
 if (!file.exists("expert_raw.csv") || !file.exists("lr_data_raw.csv") || FORCE_RERUN_EXPERT) {
  cat("  Running 00d_process_expert.R...\n")
  source("../src/r/00d_process_expert.R")
 } else {
  cat("  Loading cached expert_raw.csv and lr_data_raw.csv...\n")
 }
 expert_raw <- read_csv("expert_raw.csv", show_col_types = FALSE)
 lr_data_raw <- read_csv("lr_data_raw.csv", show_col_types = FALSE)
 cat(sprintf("  Expert: %d rows, LR: %d rows\n\n", nrow(expert_raw), nrow(lr_data_raw)))
 # ============================================================
 # Step 3b: Morgan (1976) Historical Expert Data
 # ============================================================
 cat("Step 3b: Morgan (1976) historical L-R data\n")
 # First run to generate morgan_data.csv if needed
 if (!file.exists("morgan_data.csv") || FORCE_RERUN_MORGAN) {
  cat("  Running 00e_process_morgan.R (initial processing)...\n")
  source("../src/r/00e_process_morgan.R")
 }
 # morgan_lr.csv depends on text_data.csv, so we need to check if it needs regeneration
 # It will be generated/regenerated below after text_data is created
 # ============================================================
 # Step 4: Combine Text Data Sources
 # ============================================================
 cat("Step 4: Combining text data sources\n")
 text_data <- bind_rows(manifesto, poldem_data)
 cat(sprintf("  Combined text_data: %d rows\n", nrow(text_data)))
 # Save unfiltered text_data for reproducible mismatch diagnosis
 write_csv(text_data, "text_data_unfiltered.csv")
 cat(sprintf("  Saved unfiltered text_data: %d rows, %d parties\n", nrow(text_data), n_distinct(text_data$party)))
 # ============================================================
 # Step 4b: Party Renames (applied before filtering)
 # ============================================================
 # Renames must happen BEFORE the relevance filter so that party IDs
 # match across text_data and expert_raw when computing expert coverage.
 # Simple renames only (organizational continuity: same leadership/members)
 simple_renames <- c(
  `10` = 1816L,    # DE: Greens -> Bündnis90/Grüne
  `276` = 120L,    # RO: FDSN/PDSR -> PSD (renamed 2001)
  `8054` = 878L,   # IT: PDS -> DS (renamed 1998)
  `1696` = 813L,   # IT: MSI -> AN (refounded 1995)
  `553` = 1968L,   # BE: Vlaams Blok -> Vlaams Belang (refounded 2004)
  `8058` = 1626L   # IT: Forza Italia (refounded 2013) -> Forza Italia (same party, Berlusconi)
 )
 apply_simple_renames <- function(df) {
  for (old_id in names(simple_renames)) {
    df <- df %>%
      mutate(party = ifelse(party == as.integer(old_id), simple_renames[[old_id]], party))
  }
  df
 }
 cat("\nStep 4b: Party renames\n")
 text_data <- apply_simple_renames(text_data)
 cat(sprintf("  Applied %d renames to text_data\n", length(simple_renames)))
 # ============================================================
 # Step 4c: Relevance Filter
 # ============================================================
 # Design: R pipeline filters for RELEVANCE (is this party worth modeling?).
 # Julia pipeline handles INTERPOLATION QUALITY (MAX_GAP=7 segment splitting, MIN_OBS=2).
 # Expert survey coverage is a relevance signal: CHES only covers parties with >1% vote share.
 cat("\nStep 4c: Relevance filter\n")
 parties_before <- n_distinct(text_data$party)
 # Compute expert coverage per party (with renames applied for consistent matching)
 expert_year_counts <- bind_rows(
  expert_raw %>% select(party, year),
  lr_data_raw %>% select(party, year)
 ) %>% distinct() %>%
  apply_simple_renames() %>%
  distinct() %>%
  count(party, name = "expert_years")
 expert_party_ids <- unique(expert_year_counts$party)
 cat(sprintf("  Parties with expert data: %d\n", length(expert_party_ids)))
 # Three-tier relevance filter:
 #   Tier 1: 3+ text data years (always include, regardless of expert data)
 #   Tier 2: 2 text years + any expert data (major newer parties like M5S, ANO, LREM)
 #   Tier 3: 1 text year + 3+ expert survey years (parties with rich expert coverage)
 text_data <- text_data %>%
  group_by(country, party) %>%
  mutate(n_years = n_distinct(year)) %>%
  ungroup() %>%
  left_join(expert_year_counts, by = "party") %>%
  mutate(expert_years = replace_na(expert_years, 0L)) %>%
  mutate(
    tier = case_when(
      n_years >= 3 ~ 1L,
      n_years >= 2 & party %in% expert_party_ids ~ 2L,
      n_years >= 1 & expert_years >= 3 ~ 3L,
      TRUE ~ 0L
    )
  ) %>%
  filter(tier > 0) %>%
  select(-n_years, -expert_years, -tier)
 parties_after <- n_distinct(text_data$party)
 cat(sprintf("  Parties before filter: %d\n", parties_before))
 cat(sprintf("  Parties after filter: %d\n", parties_after))
 cat(sprintf("  Parties removed: %d\n\n", parties_before - parties_after))
 # ============================================================
 # Step 5: Party Harmonization
 # ============================================================
 cat("Step 5: Party harmonization (union-aware)\n")
 # Load union mapping to identify constituent parties
 union_map <- read_csv("union_mapping.csv", show_col_types = FALSE)
 # Build set of constituent parties whose union is in text_data
 constituent_parties <- union_map %>%
  filter(manifesto_pf_id %in% unique(text_data$party)) %>%
  pull(expert_pf_id)
 cat(sprintf("  Union mappings loaded: %d rows covering %d unions\n",
            nrow(union_map), n_distinct(union_map$manifesto_pf_id)))
 cat(sprintf("  Constituent parties with unions in text_data: %d\n",
            length(unique(constituent_parties))))
 # Deduplicate union manifesto rows: where multiple CMP codes map to the same
 # union PF ID with identical content, keep only one set per (party, year, var)
 text_data_before_dedup <- nrow(text_data)
 text_data <- text_data %>%
  distinct(country, party, year, var, .keep_all = TRUE)
 cat(sprintf("  Text data: %d unique parties after harmonization\n", n_distinct(text_data$party)))
 cat(sprintf("  Text data: deduplicated %d -> %d rows\n", text_data_before_dedup, nrow(text_data)))
 # Filter expert data: keep parties in text_data OR constituent parties of unions in text_data
 expert <- expert_raw %>%
  apply_simple_renames() %>%
  group_by(country, party, var, year) %>%
  summarise(
    val = mean(val, na.rm = TRUE),
    val_int = first(val_int),
    n_scale = first(n_scale),
    n_experts = first(n_experts),
    project = first(project),
    type_low = first(type_low),
    type_high = first(type_high),
    .groups = "drop"
  ) %>%
  filter(party %in% unique(text_data$party) | party %in% constituent_parties)
 lr_data <- lr_data_raw %>%
  apply_simple_renames() %>%
  group_by(country, party, var, year) %>%
  summarise(
    val = mean(val, na.rm = TRUE),
    val_int = first(val_int),
    n_scale = first(n_scale),
    n_experts = first(n_experts),
    project = first(project),
    .groups = "drop"
  ) %>%
  filter(party %in% unique(text_data$party) | party %in% constituent_parties)
 cat(sprintf("  Expert data: %d rows (filtered to text_data parties)\n", nrow(expert)))
 cat(sprintf("  LR data (CHES/POPPA): %d rows (filtered to text_data parties)\n", nrow(lr_data)))
 # ============================================================
 # Step 5b: Integrate Morgan L-R Data
 # ============================================================
 cat("\nStep 5b: Morgan L-R data integration\n")
 # Generate morgan_lr.csv (requires text_data.csv to exist)
 # We need to regenerate it if text_data changed or if forced
 if (!file.exists("morgan_lr.csv") || FORCE_RERUN_MORGAN) {
  cat("  Generating morgan_lr.csv...\n")
  # Write text_data first so morgan script can use it
  write_csv(text_data, "text_data.csv")
  source("../src/r/00e_process_morgan.R")
 }
 # Load and integrate Morgan L-R data
 if (file.exists("morgan_lr.csv")) {
  morgan_lr <- read_csv("morgan_lr.csv", show_col_types = FALSE) %>%
    apply_simple_renames() %>%
    filter(party %in% unique(text_data$party) | party %in% constituent_parties)
  cat(sprintf("  Morgan L-R: %d rows (filtered to text_data parties)\n", nrow(morgan_lr)))
  cat(sprintf("  Morgan parties: %d\n", n_distinct(morgan_lr$party)))
  cat(sprintf("  Morgan year range: %d-%d\n", min(morgan_lr$year), max(morgan_lr$year)))
  # Combine with existing lr_data
  lr_data_before <- nrow(lr_data)
  lr_data <- bind_rows(lr_data, morgan_lr) %>%
    arrange(country, party, year, var)
  cat(sprintf("  Combined LR data: %d rows (+%d from Morgan)\n",
              nrow(lr_data), nrow(lr_data) - lr_data_before))
 } else {
  cat("  Warning: morgan_lr.csv not found, skipping Morgan integration\n")
 }
 cat("\n")
 # ============================================================
 # Step 6: Write Final Outputs
 # ============================================================
 cat("Step 6: Writing final outputs\n")
 write_csv(text_data, "text_data.csv")
 write_csv(expert, "expert.csv")
 write_csv(lr_data, "lr_data.csv")
 cat("\n============================================================\n")
 cat("Pipeline Complete!\n")
 cat("============================================================\n\n")
 cat("Output files written:\n")
 cat(sprintf("  text_data.csv: %d rows\n", nrow(text_data)))
 cat(sprintf("    - Manifesto: %d rows\n", sum(grepl("_manifesto", text_data$var))))
 cat(sprintf("    - PolDem:    %d rows\n", sum(grepl("_poldem", text_data$var))))
 cat(sprintf("  expert.csv: %d rows\n", nrow(expert)))
 cat(sprintf("  lr_data.csv: %d rows\n", nrow(lr_data)))
 cat(sprintf("    - CHES:   %d rows\n", sum(lr_data$var == "lr_ches")))
 cat(sprintf("    - POPPA:  %d rows\n", sum(lr_data$var == "lr_poppa")))
 cat(sprintf("    - Morgan: %d rows\n", sum(lr_data$var == "lr_morgan")))
 cat("\nUnique parties in text_data:", n_distinct(text_data$party), "\n")
 cat("Countries:", paste(sort(unique(text_data$country)), collapse = ", "), "\n")
 cat("Year range:", min(text_data$year, na.rm = TRUE), "-", max(text_data$year, na.rm = TRUE), "\n")
@@ -1,178 +0,0 @@
 # ============================================================
 # 00a_process_manifesto.R - Manifesto Project Data Processing
 # ============================================================
 # Processes Manifesto Project data for the 4D latent trait model
 # Input: $PARTY2D_RAW_DATA_DIR/manifesto/MPDataset_MPDS2025a.csv
 # Output: manifesto_data.csv
 # ============================================================
 library(tidyverse)
 library(countrycode)
 library(purrr)
 # Set working directory (works both in RStudio and command line)
 if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
 }
 cat("Processing Manifesto Project data...\n")
 raw_data_dir <- Sys.getenv(
  "PARTY2D_RAW_DATA_DIR",
  unset = file.path("..", "..", "_local", "raw")
 )
 manifesto_raw_path <- file.path(raw_data_dir, "manifesto", "MPDataset_MPDS2025a.csv")
 # ============================================================
 # PartyFacts Linkage
 # ============================================================
 partyfacts_raw <- read_csv('partyfacts-external-parties.csv', show_col_types = FALSE)
 manifesto_link <- partyfacts_raw %>%
  filter(dataset_key == "manifesto") %>%
  transmute(id = dataset_party_id,
            country = countrycode(country, origin = 'iso3c', destination = "iso2c"),
            party = partyfacts_id,
            party = ifelse(party == 622, 604, party))
 # ============================================================
 # Load Manifesto Data
 # ============================================================
 manifesto_data <- read_csv(manifesto_raw_path, show_col_types = FALSE)
 # ============================================================
 # CMP Code Mapping to 4 Dimensions
 # ============================================================
 vars <- tribble(
  ~type,          ~subtype,                 ~per_var,  ~stance,    ~label,
  # pro_market
  "pro_market",   "Market Regulation",      "per401",  "Positive", "Free Market Economy",
  "pro_market",   "Economic Liberalization","per402",  "Positive", "Incentives: Positive",
  "pro_market",   "Market Regulation",      "per407",  "Positive", "Protectionism: Negative",
  "pro_market",   "Economic Liberalization","per414",  "Positive", "Economic Orthodoxy",
  "pro_market",   "Economic Liberalization","per505",  "Positive", "Welfare State Limitation",
  "pro_market",   "Economic Liberalization","per507",  "Positive", "Education Limitation",
  "pro_market",   "Economic Liberalization","per702",  "Positive", "Labour Groups: Negative",
  "pro_market",   "Market Regulation",      "per406",  "Negative", "Protectionism: Positive",
  "pro_market",   "Market Regulation",      "per412",  "Negative", "Controlled Economy",
  "pro_market",   "Economic Liberalization","per504",  "Negative", "Welfare State Expansion",
  # pro_welfare
  "pro_welfare",  "Economic Intervention",  "per403",  "Positive", "Market Regulation",
  "pro_welfare",  "Economic Intervention",  "per404",  "Positive", "Economic Planning",
  "pro_welfare",  "Economic Intervention",  "per412",  "Positive", "Controlled Economy",
  "pro_welfare",  "Economic Intervention",  "per413",  "Positive", "Nationalisation",
  "pro_welfare",  "Social Services",        "per504",  "Positive", "Welfare State Expansion",
  "pro_welfare",  "Social Services",        "per506",  "Positive", "Education Expansion",
  "pro_welfare",  "Economic Intervention",  "per701",  "Positive", "Labour Groups: Positive",
  "pro_welfare",  "Economic Intervention",  "per401",  "Negative", "Free Market Economy",
  "pro_welfare",  "Social Services",        "per505",  "Negative", "Welfare State Limitation",
  # cosmopolitan
  "cosmopolitan", "Internationalism",       "per107",  "Positive", "Internationalism: Positive",
  "cosmopolitan", "Internationalism",       "per108",  "Positive", "European Community/Union: Positive",
  "cosmopolitan", "Multiculturalism",       "per607",  "Positive", "Multiculturalism: Positive",
  "cosmopolitan", "Multiculturalism",       "per201",  "Positive", "Freedom and Human Rights",
  "cosmopolitan", "Multiculturalism",       "per604",  "Positive", "traditional Morality: Negative",
  "cosmopolitan", "Internationalism",       "per109",  "Negative", "Internationalism: Negative",
  "cosmopolitan", "Multiculturalism",       "per601",  "Negative", "National Way of Life: Positive",
  # traditional
  "traditional",  "National Identity",      "per109",  "Positive", "Internationalism: Negative",
  "traditional",  "Conservative Morality",  "per110",  "Positive", "European Community/Union: Negative",
  "traditional",  "National Identity",      "per601",  "Positive", "National Way of Life: Positive",
  "traditional",  "Conservative Morality",  "per603",  "Positive", "traditional Morality: Positive",
  "traditional",  "Conservative Morality",  "per608",  "Positive", "Multiculturalism: Negative",
  "traditional",  "Conservative Morality",  "per605",  "Positive", "Law and Order: Positive",
  "traditional",  "National Identity",      "per107",  "Negative", "Internationalism: Positive",
  "traditional",  "Conservative Morality",  "per607",  "Negative", "Multiculturalism: Positive"
 )
 # ============================================================
 # Process Manifesto Data
 # ============================================================
 manifesto <- vars %>%
  pmap_dfr(~ manifesto_data %>%
             transmute(country = countrycode(countryname, origin = 'country.name', destination = 'iso2c'),
                       year = as.numeric(format(as.Date(edate, format = "%d/%m/%Y"), "%Y")),
                       id = as.character(party),
                       count = round(.data[[..3]]),
                       var = ..3,
                       label = ..5,
                       type = ..1,
                       subtype = ..2,
                       stance = ..4,
                       project = 'Manifesto Project') %>%
             left_join(manifesto_link, by = c("id", "country")) %>%
             select(-id)) %>%
  group_by(party, country, year, subtype) %>%
  summarise(
    positive = sum(count[stance == "Positive"], na.rm = TRUE),
    sample = sum(count, na.rm = TRUE),
    type = first(type),
    project = first(project),
    .groups = "drop"
  ) %>%
  na.omit() %>%
  rename(var = subtype) %>%
  # Convert to bipolar bridge structure (type_high/type_low)
  mutate(
    type_high = case_when(
      type == "pro_welfare" ~ "pro_welfare",
      type == "pro_market" ~ "pro_market",
      type == "cosmopolitan" ~ "cosmopolitan",
      type == "traditional" ~ "traditional"
    ),
    type_low = case_when(
      type %in% c("pro_welfare", "pro_market") ~ ifelse(type == "pro_welfare", "pro_market", "pro_welfare"),
      type %in% c("cosmopolitan", "traditional") ~ ifelse(type == "cosmopolitan", "traditional", "cosmopolitan")
    )
  ) %>%
  select(-type) %>%
  # Add _manifesto suffix to variable names
  mutate(var = paste0(tolower(gsub(" ", "_", var)), "_manifesto"))
 # ============================================================
 # NOTE: Temporal continuity filter moved to 00_data-management.R
 # This allows exempting parties that appear in parliamentary data
 # (parties in parliament are by definition not fringe parties)
 # ============================================================
 cat("Skipping temporal filter (applied in 00_data-management.R after combining with parl data)\n")
 cat(sprintf("  Parties: %d\n", n_distinct(manifesto$party)))
 # ============================================================
 # Write Output
 # ============================================================
 write_csv(manifesto, "manifesto_data.csv")
 cat(sprintf("Output: manifesto_data.csv (%d rows, %d parties)\n",
            nrow(manifesto), n_distinct(manifesto$party)))
 # ============================================================
 # Election Data Extraction (vote shares)
 # ============================================================
 cat("\nExtracting election data (pervote)...\n")
 election_data <- manifesto_data %>%
  transmute(
    country = countrycode(countryname, origin = 'country.name', destination = 'iso2c'),
    year = as.numeric(format(as.Date(edate, format = "%d/%m/%Y"), "%Y")),
    id = as.character(party),
    pervote = pervote
  ) %>%
  left_join(manifesto_link, by = c("id", "country")) %>%
  select(-id) %>%
  filter(!is.na(party), !is.na(pervote)) %>%
  # Keep one row per (party, country, year) — take max pervote if duplicates
  group_by(party, country, year) %>%
  summarise(pervote = max(pervote, na.rm = TRUE), .groups = "drop") %>%
  arrange(country, party, year)
 write_csv(election_data, "election_data.csv")
 cat(sprintf("Output: election_data.csv (%d rows, %d parties)\n",
            nrow(election_data), n_distinct(election_data$party)))
 # Export manifesto_link for use by other scripts
 # (poldem also needs it for CMP linkage)
@@ -1,161 +0,0 @@
 # ============================================================
 # 00c_process_poldem.R - PolDem Media Data Processing
 # ============================================================
 # Processes PolDem (Political Deliberation in the Media) data
 # for the 4D latent trait model
 #
 # Input: $PARTY2D_RAW_DATA_DIR/poldem/poldem-election_all.csv (sentence-level)
 # Output: poldem_data.csv (party-year-var aggregates)
 # ============================================================
 library(tidyverse)
 library(countrycode)
 # Set working directory (works both in RStudio and command line)
 if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
 }
 cat("Processing PolDem media data...\n")
 raw_data_dir <- Sys.getenv(
  "PARTY2D_RAW_DATA_DIR",
  unset = file.path("..", "..", "_local", "raw")
 )
 poldem_raw_path <- file.path(raw_data_dir, "poldem", "poldem-election_all.csv")
 # ============================================================
 # PartyFacts Linkage (via CMP party IDs)
 # ============================================================
 partyfacts_raw <- read_csv('partyfacts-external-parties.csv', show_col_types = FALSE)
 manifesto_link <- partyfacts_raw %>%
  filter(dataset_key == "manifesto") %>%
  transmute(cmp = as.numeric(dataset_party_id),  # Convert to numeric for join
            country_pf = countrycode(country, origin = 'iso3c', destination = "iso2c"),
            party = partyfacts_id,
            party = ifelse(party == 622, 604, party))
 # ============================================================
 # Issue Category Mapping to 4 Dimensions
 # ============================================================
 # For positive direction: type_high is the active trait
 # For negative direction: we flip (same data, just contributes to the opposite trait)
 poldem_mapping <- tribble(
  ~issue_cat,     ~dimension,  ~type_high,      ~type_low,
  # Economic dimension
  "ecolib",       "economic",  "pro_market",    "pro_welfare",   # Economic liberalization
  "welfare",      "economic",  "pro_welfare",   "pro_market",    # Welfare state
  # Final exclusion: the PolDem economic-reform category is intentionally
  # omitted because item-response diagnostics showed that it did not load
  # substantively onto the economic latent trait.
  # Cultural dimension
  "immig",        "cultural",  "cosmopolitan",  "traditional",   # Immigration (pro = cosmopolitan)
  "cultlib",      "cultural",  "cosmopolitan",  "traditional",   # Cultural liberalism
  "nationalism",  "cultural",  "traditional",   "cosmopolitan",  # Nationalism (pro = traditional)
  "europe",       "cultural",  "cosmopolitan",  "traditional",   # EU integration (pro = cosmopolitan)
  "euro",         "cultural",  "cosmopolitan",  "traditional",   # Euro currency (pro = cosmopolitan)
  "defense",      "cultural",  "traditional",   "cosmopolitan",  # Defense (pro = traditional)
  "security",     "cultural",  "traditional",   "cosmopolitan"   # Security/law-order (pro = traditional)
 )
 cat(sprintf("  Using %d issue categories\n", nrow(poldem_mapping)))
 # ============================================================
 # Load and Clean PolDem Data
 # ============================================================
 poldem_raw <- read_csv(poldem_raw_path, show_col_types = FALSE)
 cat(sprintf("  Raw PolDem data: %d rows\n", nrow(poldem_raw)))
 poldem <- poldem_raw %>%
  # Fix country codes
  mutate(country = case_when(
    iso2code == "AU" ~ "AT",  # Austria (PolDem uses AU instead of AT)
    iso2code == "UK" ~ "GB",  # United Kingdom
    TRUE ~ iso2code
  )) %>%
  # Extract year from article date (format: YYYY-MM-DD)
  mutate(year = suppressWarnings(as.numeric(substr(date_art, 1, 4)))) %>%
  # Filter to valid issue categories only
  filter(issue_cat %in% poldem_mapping$issue_cat) %>%
  # Convert direction to numeric and filter out neutral/NA
  mutate(direction = as.numeric(direction)) %>%
  filter(!is.na(direction) & direction != 0) %>%
  # Remove rows with invalid years
  filter(!is.na(year))
 cat(sprintf("  After filtering: %d rows (valid issues, non-neutral)\n", nrow(poldem)))
 # ============================================================
 # Link to PartyFacts via CMP codes
 # ============================================================
 poldem <- poldem %>%
  mutate(cmp = as.numeric(cmp)) %>%
  left_join(manifesto_link, by = "cmp") %>%
  filter(!is.na(party))
 # Report linkage
 n_linked <- nrow(poldem)
 n_unlinked <- nrow(poldem_raw %>%
                     filter(issue_cat %in% poldem_mapping$issue_cat) %>%
                     mutate(direction = as.numeric(direction)) %>%
                     filter(!is.na(direction) & direction != 0)) - n_linked
 cat(sprintf("  Linked to PartyFacts: %d rows\n", n_linked))
 if (n_unlinked > 0) {
  cat(sprintf("  Warning: %d rows could not be linked (missing CMP mapping)\n", n_unlinked))
 }
 # ============================================================
 # Aggregate to Party-Year-Issue Level
 # Using round(sum()) for weak direction values (0.5, -0.5)
 # ============================================================
 poldem_agg <- poldem %>%
  left_join(poldem_mapping, by = "issue_cat") %>%
  group_by(party, country, year, issue_cat, type_high, type_low) %>%
  summarise(
    # Sum positive directions (0.5 and 1), then round
    positive = round(sum(direction[direction > 0])),
    # Sum absolute directions for sample (all non-neutral), then round
    sample = round(sum(abs(direction))),
    n_obs = n(),
    .groups = "drop"
  ) %>%
  # Minimum 3 observations per group
  filter(n_obs >= 3) %>%
  select(-n_obs)
 cat(sprintf("  After aggregation: %d party-year-issue observations\n", nrow(poldem_agg)))
 # ============================================================
 # Format Output (matching manifesto structure)
 # ============================================================
 poldem_data <- poldem_agg %>%
  mutate(
    var = paste0(issue_cat, "_poldem"),
    project = "PolDem"
  ) %>%
  select(party, country, year, var, positive, sample, type_high, type_low, project)
 # ============================================================
 # Write Output
 # ============================================================
 write_csv(poldem_data, "poldem_data.csv")
 cat(sprintf("\nOutput: poldem_data.csv\n"))
 cat(sprintf("  Total rows: %d\n", nrow(poldem_data)))
 cat(sprintf("  Unique parties: %d\n", n_distinct(poldem_data$party)))
 cat(sprintf("  Countries: %s\n", paste(sort(unique(poldem_data$country)), collapse = ", ")))
 cat(sprintf("  Year range: %d-%d\n", min(poldem_data$year, na.rm = TRUE), max(poldem_data$year, na.rm = TRUE)))
 cat("\n  Rows by issue category:\n")
 poldem_data %>%
  group_by(var) %>%
  summarise(n = n(), .groups = "drop") %>%
  arrange(desc(n)) %>%
  print()
@@ -1,580 +0,0 @@
 # ============================================================
 # 00d_process_expert.R - Expert Survey Data Processing
 # ============================================================
 # Processes expert survey data from multiple sources:
 # - Chapel Hill Expert Survey (CHES)
 # - V-Party Dataset
 # - POPPA
 # - GPS (Norris)
 #
 # Outputs: expert_raw.csv, lr_data_raw.csv
 #
 # V5 changes:
 # - val_int (integer rounded to nearest scale point) and n_scale columns
 # - n_experts column preserved (not dropped)
 # - V-Party cultural expansion: 5 native items replace GPS ep_v6_lib_cons
 # - V-Party economic expansion: v2pawelf added
 # - Reverse-coding for V-Party cultural + welfare items
 # ============================================================
 library(tidyverse)
 library(countrycode)
 library(haven)
 library(foreign)
 # Set working directory (works both in RStudio and command line)
 if (interactive() && requireNamespace("rstudioapi", quietly = TRUE)) {
  try(setwd(dirname(rstudioapi::getActiveDocumentContext()$path)), silent = TRUE)
 }
 cat("Processing expert survey data...\n")
 # ============================================================
 # PartyFacts Linkage for CHES
 # ============================================================
 partyfacts_raw <- read_csv('partyfacts-external-parties.csv', show_col_types = FALSE)
 ches_link <- partyfacts_raw %>%
  filter(dataset_key == "ches") %>%
  transmute(id = dataset_party_id,
            country = countrycode(country, origin = 'iso3c', destination = "iso2c"),
            party = partyfacts_id)
 # ============================================================
 # Expert Count Tables (from individual response files)
 # ============================================================
 cat("  Loading expert count tables from individual response files...\n")
 # CHES 2024: dual lookup (party_id primary, country+name fallback for ID mismatches)
 ches24_exp_raw <- read_csv('~/data/ches/CHES_2024_expert_level.csv', show_col_types = FALSE)
 ches24_exp_by_id <- ches24_exp_raw %>%
  group_by(party_id) %>%
  summarise(n_experts_id = as.integer(n_distinct(id)), .groups = "drop")
 ches24_exp_by_name <- ches24_exp_raw %>%
  mutate(country_iso2 = countrycode(cname, origin = "country.name", destination = "iso2c")) %>%
  group_by(country_iso2, party_name) %>%
  summarise(n_experts_name = as.integer(n_distinct(id)), .groups = "drop")
 ches_ca_expert_counts <- read_csv('~/data/ches/CHES_CA2023_expert_level.csv', show_col_types = FALSE) %>%
  group_by(party_id) %>%
  summarise(n_experts = as.integer(n_distinct(expert)), .groups = "drop")
 ches_la_expert_counts <- read_csv('~/data/ches/CHES_LA2020_expert_level.csv', show_col_types = FALSE) %>%
  group_by(party_id) %>%
  summarise(n_experts = as.integer(n_distinct(expert_id)), .groups = "drop")
 ches_il_expert_counts <- read_csv('~/data/ches/CHES_IL_expert_level.csv', show_col_types = FALSE) %>%
  group_by(party_id, year) %>%
  summarise(n_experts = as.integer(n_distinct(id)), .groups = "drop")
 # ============================================================
 # Chapel Hill Expert Survey (CHES) - 1999-2019
 # ============================================================
 cat("  Processing CHES 1999-2019...\n")
 ches <- read_csv('~/data/ches/1999-2019_CHES_dataset_means(v3).csv', show_col_types = FALSE) %>%
  rename(country_id = country) %>%
  left_join(readRDS('~/data/ches/link.rds'), by = "country_id") %>%
  transmute(country = countrycode(country, origin = 'country.name', destination = 'iso2c'),
            vote = vote,
            year = year,
            id = as.character(party_id),
            project = 'CHES',
            n_experts = as.integer(expert),
            lrecon_ches = lrecon/10,
            galtan_ches = galtan/10) %>%
  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
  mutate(n_scale = 10L) %>%
  left_join(ches_link, by = c("id", "country")) %>%
  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
  select(-id) %>%
  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
 # ============================================================
 # CHES 2024 Update
 # ============================================================
 cat("  Processing CHES 2024...\n")
 # Country code lookup for CHES 2024 format
 country_lookup <- c(
  "be" = "BE", "dk" = "DK", "ge" = "DE", "gr" = "GR", "esp" = "ES",
  "fr" = "FR", "irl" = "IE", "it" = "IT", "nl" = "NL", "uk" = "GB",
  "por" = "PT", "aus" = "AT", "fin" = "FI", "sv" = "SE", "bul" = "BG",
  "cz" = "CZ", "est" = "EE", "hun" = "HU", "lat" = "LV", "lith" = "LT",
  "pol" = "PL", "rom" = "RO", "slo" = "SK", "sle" = "SI", "cro" = "HR",
  "tur" = "TR", "nor" = "NO", "swi" = "CH", "mal" = "MT", "cyp" = "CY",
  "ice" = "IS"
 )
 convert_country_codes <- function(codes) {
  result <- country_lookup[codes]
  result[is.na(result)] <- codes[is.na(result)]
  return(unname(result))
 }
 ches24 <- read_csv('~/data/ches/CHES_2024_final_v2.csv', show_col_types = FALSE) %>%
  mutate(country_iso2 = convert_country_codes(country)) %>%
  left_join(ches24_exp_by_id, by = "party_id") %>%
  left_join(ches24_exp_by_name, by = c("country_iso2", "party" = "party_name")) %>%
  transmute(country = country_iso2,
            vote = vote,
            year = 2024,
            id = as.character(party_id),
            project = 'CHES',
            n_experts = coalesce(n_experts_id, n_experts_name),
            lrecon_ches = lrecon/10,
            galtan_ches = galtan/10) %>%
  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
  mutate(n_scale = 10L) %>%
  left_join(ches_link, by = c("id", "country")) %>%
  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
  select(-id) %>%
  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
 ches <- bind_rows(ches, ches24)
 # ============================================================
 # CHES Canada 2023
 # ============================================================
 cat("  Processing CHES Canada 2023...\n")
 ches_ca <- read_csv('~/data/ches/CHES_CA2023.csv', show_col_types = FALSE) %>%
  filter(!is.na(partyfacts_id)) %>%
  left_join(ches_ca_expert_counts, by = "party_id") %>%
  transmute(country = "CA",
            year = 2023,
            party = partyfacts_id,
            project = 'CHES',
            n_experts = n_experts,
            lrecon_ches = lrecon/10,
            galtan_ches = galtan/10) %>%
  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
  mutate(n_scale = 10L) %>%
  filter(!is.na(val), !is.na(party)) %>%
  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
 ches <- bind_rows(ches, ches_ca)
 cat(sprintf("    CHES Canada: %d observations\n", nrow(ches_ca)))
 # ============================================================
 # CHES Latin America 2020
 # ============================================================
 cat("  Processing CHES Latin America 2020...\n")
 ches_la_link <- read_csv('~/data/ches/ches_la_link.csv', show_col_types = FALSE) %>%
  transmute(id = as.character(ches_party_id),
            party = partyfacts_id)
 ches_la <- read_csv('~/data/ches/ches_la_2020_aggregate_level_v01.csv', show_col_types = FALSE) %>%
  left_join(ches_la_expert_counts, by = "party_id") %>%
  transmute(country = country_abb,
            year = 2020,
            id = as.character(party_id),
            project = 'CHES',
            n_experts = n_experts,
            lrecon_ches = lrecon/10,
            galtan_ches = galtan/10) %>%
  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
  mutate(n_scale = 10L) %>%
  left_join(ches_la_link, by = "id") %>%
  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
  select(-id) %>%
  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
 ches <- bind_rows(ches, ches_la)
 cat(sprintf("    CHES Latin America: %d observations\n", nrow(ches_la)))
 # ============================================================
 # CHES Israel 2021-2022
 # ============================================================
 cat("  Processing CHES Israel 2021-2022...\n")
 ches_il_link <- read_csv('~/data/ches/ches_israel_link.csv', show_col_types = FALSE) %>%
  transmute(id = as.character(ches_party_id),
            party = partyfacts_id)
 ches_il <- read_csv('~/data/ches/CHES_ISRAEL_means_2021_2022.csv', show_col_types = FALSE) %>%
  left_join(ches_il_expert_counts, by = c("party_id", "year")) %>%
  transmute(country = "IL",
            year = year,
            id = as.character(party_id),
            project = 'CHES',
            n_experts = n_experts,
            lrecon_ches = lrecon/10,
            galtan_ches = galtan/10) %>%
  pivot_longer(cols = lrecon_ches:galtan_ches, names_to = 'var', values_to = 'val') %>%
  mutate(n_scale = 10L) %>%
  left_join(ches_il_link, by = "id") %>%
  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
  select(-id) %>%
  mutate(type_low = ifelse(var == "lrecon_ches", "pro_welfare", "cosmopolitan"),
         type_high = ifelse(var == "lrecon_ches", "pro_market", "traditional"))
 ches <- bind_rows(ches, ches_il)
 cat(sprintf("    CHES Israel: %d observations\n", nrow(ches_il)))
 cat(sprintf("    CHES total: %d observations\n", nrow(ches)))
 # ============================================================
 # CHES General Left-Right (for anchoring)
 # ============================================================
 cat("  Processing CHES LR anchoring data...\n")
 ches_lr <- read_csv('~/data/ches/1999-2019_CHES_dataset_means(v3).csv', show_col_types = FALSE) %>%
  rename(country_id = country) %>%
  left_join(readRDS('~/data/ches/link.rds'), by = "country_id") %>%
  transmute(country = countrycode(country, origin = 'country.name', destination = 'iso2c'),
            vote = vote,
            year = year,
            id = as.character(party_id),
            project = 'CHES',
            n_experts = as.integer(expert),
            val = lrgen/10,
            var = 'lr_ches',
            n_scale = 10L) %>%
  left_join(ches_link, by = c("id", "country")) %>%
  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
  select(-id)
 ches24_lr <- read_csv('~/data/ches/CHES_2024_final_v2.csv', show_col_types = FALSE) %>%
  mutate(country_iso2 = convert_country_codes(country)) %>%
  left_join(ches24_exp_by_id, by = "party_id") %>%
  left_join(ches24_exp_by_name, by = c("country_iso2", "party" = "party_name")) %>%
  transmute(country = country_iso2,
            vote = vote,
            year = 2024,
            id = as.character(party_id),
            project = 'CHES',
            n_experts = coalesce(n_experts_id, n_experts_name),
            val = lrgen/10,
            var = 'lr_ches',
            n_scale = 10L) %>%
  left_join(ches_link, by = c("id", "country")) %>%
  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
  select(-id)
 # CHES Canada LR
 ches_ca_lr <- read_csv('~/data/ches/CHES_CA2023.csv', show_col_types = FALSE) %>%
  filter(!is.na(partyfacts_id)) %>%
  left_join(ches_ca_expert_counts, by = "party_id") %>%
  transmute(country = "CA",
            year = 2023,
            party = partyfacts_id,
            project = 'CHES',
            n_experts = n_experts,
            val = lrgen/10,
            var = 'lr_ches',
            n_scale = 10L) %>%
  filter(!is.na(val), !is.na(party))
 # CHES Latin America LR
 ches_la_lr <- read_csv('~/data/ches/ches_la_2020_aggregate_level_v01.csv', show_col_types = FALSE) %>%
  left_join(ches_la_expert_counts, by = "party_id") %>%
  transmute(country = country_abb,
            year = 2020,
            id = as.character(party_id),
            project = 'CHES',
            n_experts = n_experts,
            val = lrgen/10,
            var = 'lr_ches',
            n_scale = 10L) %>%
  left_join(ches_la_link, by = "id") %>%
  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
  select(-id)
 # CHES Israel LR
 ches_il_lr <- read_csv('~/data/ches/CHES_ISRAEL_means_2021_2022.csv', show_col_types = FALSE) %>%
  left_join(ches_il_expert_counts, by = c("party_id", "year")) %>%
  transmute(country = "IL",
            year = year,
            id = as.character(party_id),
            project = 'CHES',
            n_experts = n_experts,
            val = lrgen/10,
            var = 'lr_ches',
            n_scale = 10L) %>%
  left_join(ches_il_link, by = "id") %>%
  filter(!is.na(val), !is.na(party), !is.na(country)) %>%
  select(-id)
 ches_lr <- bind_rows(ches_lr, ches24_lr, ches_ca_lr, ches_la_lr, ches_il_lr)
 # ============================================================
 # V-Party Dataset (V5: expanded to 7 variables)
 # ============================================================
 cat("  Processing V-Party...\n")
 vparty_raw <- readRDS('~/data/v-party/V-Dem-CPD-Party-V2.rds')
 # Economic 1: v2pariglef_osp (0-6 scale, higher = more right, NO reverse)
 vparty_econ1 <- vparty_raw %>%
  transmute(
    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
    year = year,
    party = pf_party_id,
    project = "V-Party",
    n_experts = as.integer(v2pariglef_nr),
    val = v2pariglef_osp / 6,
    val_int = as.integer(round(v2pariglef_osp)),
    n_scale = 6L,
    var = "lrecon_vparty",
    type_low = "pro_welfare",
    type_high = "pro_market"
  ) %>%
  na.omit()
 # Economic 2 (NEW): v2pawelf_osp (0-5 scale, higher = more welfare = LEFT, REVERSE)
 vparty_econ2 <- vparty_raw %>%
  transmute(
    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
    year = year,
    party = pf_party_id,
    project = "V-Party",
    n_experts = as.integer(v2pawelf_nr),
    val = 1 - v2pawelf_osp / 5,
    val_int = 5L - as.integer(round(v2pawelf_osp)),
    n_scale = 5L,
    var = "welf_vparty",
    type_low = "pro_welfare",
    type_high = "pro_market"
  ) %>%
  na.omit()
 # Cultural 1 (NEW): v2paimmig_osp (0-4 scale, higher = more pro-immigration = GAL, REVERSE)
 vparty_cult1 <- vparty_raw %>%
  transmute(
    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
    year = year,
    party = pf_party_id,
    project = "V-Party",
    n_experts = as.integer(v2paimmig_nr),
    val = 1 - v2paimmig_osp / 4,
    val_int = 4L - as.integer(round(v2paimmig_osp)),
    n_scale = 4L,
    var = "immig_vparty",
    type_low = "cosmopolitan",
    type_high = "traditional"
  ) %>%
  na.omit()
 # Cultural 2 (NEW): v2palgbt_osp (0-4 scale, higher = more pro-LGBT = GAL, REVERSE)
 vparty_cult2 <- vparty_raw %>%
  transmute(
    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
    year = year,
    party = pf_party_id,
    project = "V-Party",
    n_experts = as.integer(v2palgbt_nr),
    val = 1 - v2palgbt_osp / 4,
    val_int = 4L - as.integer(round(v2palgbt_osp)),
    n_scale = 4L,
    var = "lgbt_vparty",
    type_low = "cosmopolitan",
    type_high = "traditional"
  ) %>%
  na.omit()
 # Cultural 3 (NEW): v2paculsup_osp (0-4 scale, higher = less cultural superiority = GAL, REVERSE)
 vparty_cult3 <- vparty_raw %>%
  transmute(
    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
    year = year,
    party = pf_party_id,
    project = "V-Party",
    n_experts = as.integer(v2paculsup_nr),
    val = 1 - v2paculsup_osp / 4,
    val_int = 4L - as.integer(round(v2paculsup_osp)),
    n_scale = 4L,
    var = "culsup_vparty",
    type_low = "cosmopolitan",
    type_high = "traditional"
  ) %>%
  na.omit()
 # Cultural 4 (NEW): v2parelig_osp (0-4 scale, higher = less religious = GAL, REVERSE)
 vparty_cult4 <- vparty_raw %>%
  transmute(
    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
    year = year,
    party = pf_party_id,
    project = "V-Party",
    n_experts = as.integer(v2parelig_nr),
    val = 1 - v2parelig_osp / 4,
    val_int = 4L - as.integer(round(v2parelig_osp)),
    n_scale = 4L,
    var = "relig_vparty",
    type_low = "cosmopolitan",
    type_high = "traditional"
  ) %>%
  na.omit()
 # Cultural 5 (NEW): v2pagender_osp (0-4 scale, higher = more pro-gender equality = GAL, REVERSE)
 vparty_cult5 <- vparty_raw %>%
  transmute(
    country = countrycode(country_name, origin = "country.name", destination = "iso2c"),
    year = year,
    party = pf_party_id,
    project = "V-Party",
    n_experts = as.integer(v2pagender_nr),
    val = 1 - v2pagender_osp / 4,
    val_int = 4L - as.integer(round(v2pagender_osp)),
    n_scale = 4L,
    var = "gender_vparty",
    type_low = "cosmopolitan",
    type_high = "traditional"
  ) %>%
  na.omit()
 vparty <- bind_rows(vparty_econ1, vparty_econ2,
                    vparty_cult1, vparty_cult2, vparty_cult3,
                    vparty_cult4, vparty_cult5)
 cat(sprintf("    V-Party: %d observations (7 variables)\n", nrow(vparty)))
 cat(sprintf("      lrecon: %d, welf: %d\n", nrow(vparty_econ1), nrow(vparty_econ2)))
 cat(sprintf("      immig: %d, lgbt: %d, culsup: %d, relig: %d, gender: %d\n",
            nrow(vparty_cult1), nrow(vparty_cult2), nrow(vparty_cult3),
            nrow(vparty_cult4), nrow(vparty_cult5)))
 # ============================================================
 # POPPA Dataset
 # ============================================================
 cat("  Processing POPPA...\n")
 poppa <- readRDS('~/data/POPPA/poppa_integrated_v2.rds') %>%
  transmute(country = countrycode(country_short, origin = "iso3c", destination = "iso2c"),
            party = partyfacts_id,
            val = lrecon/10,
            var = "lrecon_poppa",
            type_low = "pro_welfare",
            type_high = "pro_market",
            n_experts = as.integer(n_experts),
            n_scale = 10L,
            year = as.numeric(sub(".*-\\s*(\\d+)", "\\1", wave)),
            project = "POPPA") %>%
  na.omit()
 cat(sprintf("    POPPA: %d observations\n", nrow(poppa)))
 # POPPA General LR
 poppa_lr <- readRDS('~/data/POPPA/poppa_integrated_v2.rds') %>%
  transmute(country = countrycode(country_short, origin = "iso3c", destination = "iso2c"),
            party = partyfacts_id,
            val = lroverall/10,
            var = "lr_poppa",
            n_experts = as.integer(n_experts),
            n_scale = 10L,
            year = as.numeric(sub(".*-\\s*(\\d+)", "\\1", wave)),
            project = "POPPA") %>%
  na.omit()
 # ============================================================
 # GPS (Norris) Survey
 # ============================================================
 cat("  Processing GPS...\n")
 gps <- read.delim("~/data/GPS_norris/Global Party Survey by Party SPSS V2_1_Apr_2020-2.tab") %>%
  transmute(n_experts = as.integer(Experts),
            lrecon_gps = as.numeric(V4_Scale)/10,
            libcon_gps = as.numeric(V6_Scale)/10,
            party = ID_PartyFacts,
            country = countrycode(ifelse(ISO == "MAC", "MKD", ISO), origin = "iso3c", destination = "iso2c"),
            year = 2019,
            n_scale = 10L,
            project = "GPS") %>%
  pivot_longer(cols = lrecon_gps:libcon_gps, names_to = 'var', values_to = 'val') %>%
  mutate(type_low = ifelse(var == "lrecon_gps", "pro_welfare", "cosmopolitan"),
         type_high = ifelse(var == "lrecon_gps", "pro_market", "traditional")) %>%
  na.omit()
 cat(sprintf("    GPS: %d observations\n", nrow(gps)))
 # ============================================================
 # Combine Expert Data
 # ============================================================
 cat("  Combining expert surveys...\n")
 expert_raw <- select(ches, -vote) %>%
  bind_rows(vparty) %>%
  bind_rows(gps) %>%
  bind_rows(poppa) %>%
  unique() %>%
  arrange(country, party, year, var) %>%
  filter(!is.na(val), !is.na(party), !is.na(country), !is.na(var))
 # Compute val_int for datasets that don't have it pre-computed
 # V-Party already has val_int; CHES/GPS/POPPA need it computed from val * n_scale
 expert_raw <- expert_raw %>%
  mutate(
    val_int = ifelse(is.na(val_int), as.integer(round(val * n_scale)), val_int),
    val_int = pmin(pmax(val_int, 0L), n_scale)
  )
 # Boundary adjustments for continuous val (avoid exact 0 or 1 for Stan prior means)
 expert_raw <- expert_raw %>%
  mutate(
    val = case_when(
      val == 0 ~ val + 1e-4,
      val == 1 ~ val - 1e-4,
      TRUE ~ val
    ))
 # ============================================================
 # Combine LR Data
 # ============================================================
 lr_data_raw <- ches_lr %>%
  bind_rows(poppa_lr) %>%
  select(-any_of("vote"))
 # Boundary adjustments for continuous val (avoid exact 0 or 1)
 lr_data_raw <- lr_data_raw %>%
  mutate(
    val = case_when(
      val == 0 ~ val + 1e-4,
      val == 1 ~ val - 1e-4,
      TRUE ~ val
    ))
 # Compute val_int for LR data
 lr_data_raw <- lr_data_raw %>%
  mutate(
    val_int = as.integer(round(val * n_scale)),
    val_int = pmin(pmax(val_int, 0L), n_scale)
  )
 # ============================================================
 # Write Outputs
 # ============================================================
 write_csv(expert_raw, "expert_raw.csv")
 write_csv(lr_data_raw, "lr_data_raw.csv")
 cat(sprintf("\nOutputs written:\n"))
 cat(sprintf("  expert_raw.csv: %d rows\n", nrow(expert_raw)))
 cat(sprintf("  lr_data_raw.csv: %d rows\n", nrow(lr_data_raw)))
 cat("\n  Expert data by source:\n")
 expert_raw %>%
  group_by(project) %>%
  summarise(n = n(), .groups = "drop") %>%
  print()
 cat("\n  New columns check:\n")
 cat(sprintf("  val_int range: %d - %d\n", min(expert_raw$val_int), max(expert_raw$val_int)))
 cat(sprintf("  n_scale values: %s\n", paste(sort(unique(expert_raw$n_scale)), collapse = ", ")))
 cat(sprintf("  n_experts non-NA: %d / %d\n", sum(!is.na(expert_raw$n_experts)), nrow(expert_raw)))
@@ -1,388 +0,0 @@
 # 00e_process_morgan.R
 # Process Morgan (1976) expert party position data
 #
 # Source: Morgan, Michael-John (1976). "The Modelling of Governmental
 # Coalition Formation: A Policy-Based Approach with Interval Measurement."
 # PhD dissertation, University of Michigan.
 #
 # Data extracted from Appendix B.3 (Tables B.3.1-B.3.12) via OCR.
 # Position scores are 25%-truncated means (midmeans) from expert surveys.
 # Scale: 0-100 (left-right)
 library(tidyverse)
 cat("Processing Morgan (1976) expert party position data...\n")
 # Load raw extracted data
 morgan_raw <- read_csv("morgan_positions_raw.csv", show_col_types = FALSE)
 cat(sprintf("Loaded %d party-period observations from %d countries\n",
            nrow(morgan_raw), n_distinct(morgan_raw$country)))
 # Load PartyFacts linkage data
 partyfacts <- read_csv("partyfacts-external-parties.csv", show_col_types = FALSE)
 # Filter to Morgan dataset entries
 morgan_pf <- partyfacts %>%
  filter(dataset_key == "morgan") %>%
  select(country, name_short, name_english, year_first, year_last,
         external_id, partyfacts_id) %>%
  rename(party_abbrev_pf = name_short)
 cat(sprintf("Found %d Morgan parties in PartyFacts\n", nrow(morgan_pf)))
 # Map extracted abbreviations to PartyFacts abbreviations
 # Some adjustments needed due to OCR/transcription differences
 abbrev_map <- tribble(
  ~country, ~party_abbrev, ~party_abbrev_pf,
  # Denmark
  "DNK", "SOCd", "SOCD",
  "DNK", "SOCL", "SOCL",
  "DNK", "COMM", "COMM",
  "DNK", "RAD", "RAD",
  "DNK", "LIB", "LIB",
  "DNK", "CONS", "CONS",
  "DNK", "LS", "LS",
  "DNK", "LC", "LC",
  "DNK", "JUST", "JUST",
  # Finland
  "FIN", "SKDL", "SKDL",
  "FIN", "SOCd", "SOCD",
  "FIN", "PROG", "PROG",
  "FIN", "AGR", "AGR",
  "FIN", "SWPP", "SWPP",
  "FIN", "CONS", "CONS",
  "FIN", "NPF", "NPF",
  "FIN", "PDEM", "PDEM",
  "FIN", "SDWS", "SDWS",
  "FIN", "CENT", "CENT",
  "FIN", "FRP", "FRP",
  "FIN", "LIB", "LIB",
  # Iceland
  "ISL", "COMM", "COMM",
  "ISL", "SOCd", "SOCD",
  "ISL", "PROG", "PROG",
  "ISL", "LIB", "LIB",
  "ISL", "INDP", "INDP",
  "ISL", "CONS", "CONS",
  "ISL", "LLIB", "LLIB",
  # Norway
  "NOR", "LAB", "LAB",
  "NOR", "LIB", "LIB",
  "NOR", "AGR", "AGR",
  "NOR", "CONS", "CONS",
  "NOR", "COMM", "COMM",
  "NOR", "SOCL", "SOCL",
  "NOR", "CHPP", "CHPP",
  "NOR", "CENT", "CENT",
  # Sweden
  "SWE", "COMM", "COMM",
  "SWE", "SOCd", "SOCD",
  "SWE", "AGR", "AGR",
  "SWE", "LIB", "LIB",
  "SWE", "CONS", "CONS",
  "SWE", "CENT", "CENT",
  # Netherlands
  "NLD", "CPN", "CPN",
  "NLD", "SOCd", "SOCD",
  "NLD", "RAD", "RAD",
  "NLD", "KVP", "KVP",
  "NLD", "CHU", "CHU",
  "NLD", "LIB", "LIB",
  "NLD", "ARP", "ARP",
  "NLD", "SGP", "SGP",
  "NLD", "NSB", "NSB",
  "NLD", "PVDA", "PVDA",
  "NLD", "VVD", "VVD",
  "NLD", "PSP", "PSP",
  "NLD", "PPR", "PPR",
  "NLD", "D66", "D66",
  "NLD", "DS70", "DS70",
  "NLD", "GPV", "GPV",
  "NLD", "BP", "BP",
  # Belgium
  "BEL", "COMM", "COMM",
  "BEL", "POB", "POB",
  "BEL", "CATH", "CATH",
  "BEL", "LIB", "LIB",
  "BEL", "FNAT", "FNAT",
  "BEL", "REX", "REX",
  "BEL", "PSB", "PSB",
  "BEL", "RW", "RW",
  "BEL", "PSC", "PSC",
  "BEL", "FDF", "FDF",
  "BEL", "VOLK", "VOLK",
  "BEL", "PLP", "PLP",
  # France (Fourth Republic)
  "FRA", "PCF", "PCF",
  "FRA", "SFIO", "SFIO",
  "FRA", "MRP", "MRP",
  "FRA", "RDA", "RDA",
  "FRA", "UDSR", "UDSR",
  "FRA", "RAD", "RAD",
  "FRA", "RS", "RS",
  "FRA", "RPF", "RPF",
  "FRA", "AR", "AR",
  "FRA", "ARS", "ARS",
  "FRA", "RI", "RI",
  "FRA", "CNIP", "CNIP",
  "FRA", "PUS", "PUS",
  "FRA", "PAYS", "PAYS",
  "FRA", "AP", "AP",
  "FRA", "PRL", "PRL",
  "FRA", "POUJ", "POUJ",
  # Weimar Germany
  "DEU", "KPD", "KPD",
  "DEU", "SDAP", "SDAP",
  "DEU", "DDP", "DDP",
  "DEU", "DZP", "DZP",
  "DEU", "BVP", "BVP",
  "DEU", "DVP", "DVP",
  "DEU", "RDMW", "RDMW",
  "DEU", "LVP", "LVP",
  "DEU", "DNVP", "DNVP",
  "DEU", "NAZI", "NAZI",
  # Italy
  "ITA", "PCI", "PCI",
  "ITA", "PSIU", "PSIU",
  "ITA", "PSI", "PSI",
  "ITA", "PSDI", "PSDI",
  "ITA", "PRI", "PRI",
  "ITA", "DC", "DC",
  "ITA", "PLI", "PLI",
  "ITA", "MON", "MON",
  "ITA", "MSI", "MSI",
  # Luxembourg
  "LUX", "COMM", "COMM",
  "LUX", "SOCd", "SOCD",
  "LUX", "CSOC", "CSOC",
  "LUX", "GRPD", "GRPD",
  # Israel
  "ISR", "RAKA", "RAKA",
  "ISR", "MAKI", "MAKI",
  "ISR", "MAPM", "MAPM",
  "ISR", "MADT", "MADT",
  "ISR", "ADUT", "ADUT",
  "ISR", "MAAR", "MAAR",
  "ISR", "LAB", "LAB",
  "ISR", "MAPI", "MAPI",
  "ISR", "PAUG", "PAUG",
  "ISR", "RAFI", "RAFI",
  "ISR", "PROG", "PROG",
  "ISR", "ILIB", "ILIB",
  "ISR", "NRP", "NRP",
  "ISR", "URF", "URF",
  "ISR", "LIB", "LIB",
  "ISR", "NATL", "NATL",
  "ISR", "TORA", "TORA",
  "ISR", "LIKD", "LIKD",
  "ISR", "ZION", "ZION",
  "ISR", "GHAL", "GHAL",
  "ISR", "AGDT", "AGDT",
  "ISR", "HRUT", "HRUT"
 )
 # Some parties in raw data that don't have exact matches - need special handling
 # (e.g., parties that only exist in one period in PartyFacts but appear in both)
 # We'll join using the period-based matching
 # Expand periods to years for matching
 morgan_expanded <- morgan_raw %>%
  mutate(
    year_start = as.integer(str_extract(period, "^\\d{4}")),
    year_end = as.integer(str_extract(period, "\\d{4}$"))
  )
 # Join with abbreviation map
 morgan_mapped <- morgan_expanded %>%
  left_join(abbrev_map, by = c("country", "party_abbrev"))
 # Check for unmatched abbreviations
 unmatched_abbrev <- morgan_mapped %>%
  filter(is.na(party_abbrev_pf)) %>%
  distinct(country, party_abbrev)
 if (nrow(unmatched_abbrev) > 0) {
  cat("\nWarning: Unmatched abbreviations:\n")
  print(unmatched_abbrev)
 }
 # Join with PartyFacts
 morgan_joined <- morgan_mapped %>%
  left_join(morgan_pf, by = c("country", "party_abbrev_pf")) %>%
  # For parties with overlapping periods, use period overlap
  mutate(
    period_overlap = pmax(0,
      pmin(year_end, year_last) - pmax(year_start, year_first) + 1)
  ) %>%
  # Keep best match per party-period (max overlap)
  group_by(country, party_abbrev, period) %>%
  slice_max(period_overlap, n = 1, with_ties = FALSE) %>%
  ungroup()
 # Check for unmatched parties
 unmatched <- morgan_joined %>%
  filter(is.na(partyfacts_id)) %>%
  distinct(country, party_abbrev, party_name, period)
 if (nrow(unmatched) > 0) {
  cat(sprintf("\n%d party-periods without PartyFacts match:\n", nrow(unmatched)))
  print(unmatched)
 }
 # Dedup: when multiple abbreviations map to the same PF ID, keep only one
 matched <- morgan_joined %>%
  filter(!is.na(partyfacts_id)) %>%
  group_by(country, partyfacts_id, period) %>%
  slice(1) %>%
  ungroup()
 cat(sprintf("\nMatched %d of %d party-period observations (%.1f%%)\n",
            nrow(matched), nrow(morgan_raw),
            100 * nrow(matched) / nrow(morgan_raw)))
 # Normalize position to [0,1] scale
 # Original scale: 0-100
 # Apply boundary adjustments like other expert data
 eps <- 0.005
 morgan_processed <- matched %>%
  mutate(
    # Normalize to [0,1]
    lr_morgan = position / 100,
    # Apply boundary adjustments
    lr_morgan = case_when(
      lr_morgan <= 0 ~ eps,
      lr_morgan >= 1 ~ 1 - eps,
      TRUE ~ lr_morgan
    ),
    # Calculate standard error (sd / sqrt(n))
    lr_morgan_se = (sd / 100) / sqrt(n_surveys),
    # Set minimum SE for extreme parties (sd=0)
    lr_morgan_se = pmax(lr_morgan_se, 0.01)
  ) %>%
  select(
    country,
    partyfacts_id,
    period,
    year_start,
    year_end,
    party_abbrev,
    party_name,
    lr_morgan,
    lr_morgan_se,
    n_surveys
  ) %>%
  arrange(country, year_start, lr_morgan)
 # Summary statistics
 cat("\nSummary of processed Morgan data:\n")
 cat(sprintf("  Countries: %d\n", n_distinct(morgan_processed$country)))
 cat(sprintf("  Parties: %d\n", n_distinct(morgan_processed$partyfacts_id)))
 cat(sprintf("  Observations: %d\n", nrow(morgan_processed)))
 # Distribution of positions
 cat("\nPosition distribution:\n")
 print(summary(morgan_processed$lr_morgan))
 # Write output
 write_csv(morgan_processed, "morgan_data.csv")
 cat(sprintf("\nWrote morgan_data.csv with %d rows\n", nrow(morgan_processed)))
 # Also provide a summary by country and period
 summary_by_country <- morgan_processed %>%
  group_by(country, period) %>%
  summarise(
    n_parties = n(),
    mean_pos = mean(lr_morgan),
    sd_pos = sd(lr_morgan),
    .groups = "drop"
  )
 cat("\nSummary by country and period:\n")
 print(summary_by_country, n = 50)
 # ============================================================
 # Generate lr_data-compatible output for pipeline integration
 # ============================================================
 cat("\n============================================================\n")
 cat("Generating lr_data-compatible output (postwar only)\n")
 cat("============================================================\n")
 # Load text_data to get party-years with manifesto/PolDem coverage
 text_data <- read_csv("text_data.csv", show_col_types = FALSE)
 # Convert Morgan ISO3 country codes to ISO2 (matching text_data format)
 iso3_to_iso2 <- c(
  "DNK" = "DK", "FIN" = "FI", "ISL" = "IS", "NOR" = "NO", "SWE" = "SE",
  "NLD" = "NL", "BEL" = "BE", "DEU" = "DE", "FRA" = "FR", "ITA" = "IT",
  "LUX" = "LU", "ISR" = "IL"
 )
 # Filter to postwar periods only (1945+)
 morgan_postwar <- morgan_processed %>%
  filter(year_end >= 1945) %>%
  mutate(country_iso2 = iso3_to_iso2[country])
 cat(sprintf("Postwar Morgan observations: %d party-periods\n", nrow(morgan_postwar)))
 cat(sprintf("Countries: %s\n", paste(unique(morgan_postwar$country_iso2), collapse = ", ")))
 # Get unique party-years from text_data
 text_party_years <- text_data %>%
  select(party, country, year) %>%
  distinct()
 cat(sprintf("Unique party-years in text_data: %d\n", nrow(text_party_years)))
 # For each Morgan party-period, expand to all years where that party has text data
 # within the Morgan period range (1945-1973 for postwar)
 morgan_lr <- morgan_postwar %>%
  # Join with text_data party-years
  # Many-to-many is expected: one Morgan party-period maps to multiple years
  inner_join(
    text_party_years,
    by = c("partyfacts_id" = "party", "country_iso2" = "country"),
    relationship = "many-to-many"
  ) %>%
  # Keep only years within the Morgan period
  filter(year >= year_start & year <= year_end) %>%
  # Format for lr_data.csv compatibility
  transmute(
    country = country_iso2,
    party = partyfacts_id,
    var = "lr_morgan",
    year = year,
    val = lr_morgan,
    project = "Morgan",
    # Morgan's continuous 0-100 scale is discretized to 10 points (matching CHES
    # resolution) with the actual number of experts. The reconstructed sum
    # round(mean × K × 10) is analogous to how CHES means are handled.
    # See docs/k_scaling_validation.md Section 4.
    n_scale = 10L,
    val_int = as.integer(round(lr_morgan * 10)),
    n_experts = as.integer(n_surveys)
  ) %>%
  distinct() %>%
  arrange(country, party, year)
 cat(sprintf("\nGenerated %d lr_morgan observations\n", nrow(morgan_lr)))
 cat(sprintf("  Unique parties: %d\n", n_distinct(morgan_lr$party)))
 cat(sprintf("  Year range: %d-%d\n", min(morgan_lr$year), max(morgan_lr$year)))
 # Summary by country
 morgan_lr_summary <- morgan_lr %>%
  group_by(country) %>%
  summarise(
    n_parties = n_distinct(party),
    n_obs = n(),
    year_min = min(year),
    year_max = max(year),
    .groups = "drop"
  )
 cat("\nMorgan L-R data by country:\n")
 print(morgan_lr_summary, n = 20)
 # Write morgan_lr.csv
 write_csv(morgan_lr, "morgan_lr.csv")
 cat(sprintf("\nWrote morgan_lr.csv with %d rows\n", nrow(morgan_lr)))