feat: add right-wing party axis validation
- Add CANONICAL_RIGHT (PVV, FVD, JA21, SGP) and CANONICAL_LEFT frozensets to analysis/config.py as the canonical source of truth - Update analysis/svd_labels.py to import from config; re-export as RIGHT_PARTIES/LEFT_PARTIES for backward compatibility - Add build_window_party_scores helper to analysis/explorer_data.py - Add 7 integration tests in tests/test_axis_political_orientation.py validating that canonical right parties appear on the right side of SVD axes (x=component 1, y=component 2) using real DuckDB data
This commit is contained in:
@@ -0,0 +1,280 @@
|
||||
"""Configuration constants for the parliamentary explorer.
|
||||
|
||||
This module contains all constant definitions used across the explorer.
|
||||
It is intentionally free of Streamlit and DuckDB dependencies.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Dict
|
||||
|
||||
__all__ = [
|
||||
"PARTY_COLOURS",
|
||||
"SVD_THEMES",
|
||||
"KNOWN_MAJOR_PARTIES",
|
||||
"CURRENT_PARLIAMENT_PARTIES",
|
||||
"_PARTY_NORMALIZE",
|
||||
"CANONICAL_RIGHT",
|
||||
"CANONICAL_LEFT",
|
||||
]
|
||||
|
||||
CANONICAL_RIGHT: frozenset[str] = frozenset(
|
||||
{
|
||||
"PVV",
|
||||
"FVD",
|
||||
"JA21",
|
||||
"SGP",
|
||||
}
|
||||
)
|
||||
|
||||
CANONICAL_LEFT: frozenset[str] = frozenset(
|
||||
{
|
||||
"SP",
|
||||
"PvdA",
|
||||
"GL",
|
||||
"GroenLinks",
|
||||
"GroenLinks-PvdA",
|
||||
"DENK",
|
||||
"PvdD",
|
||||
"Volt",
|
||||
}
|
||||
)
|
||||
|
||||
PARTY_COLOURS: Dict[str, str] = {
|
||||
"VVD": "#1E73BE",
|
||||
"PVV": "#002366",
|
||||
"D66": "#00A36C",
|
||||
"CDA": "#4CAF50",
|
||||
"SP": "#E53935",
|
||||
"PvdA": "#D32F2F",
|
||||
"GroenLinks": "#388E3C",
|
||||
"GroenLinks-PvdA": "#2E7D32",
|
||||
"CU": "#0288D1",
|
||||
"SGP": "#F4511E",
|
||||
"PvdD": "#43A047",
|
||||
"FVD": "#6A1B9A",
|
||||
"JA21": "#7B1FA2",
|
||||
"BBB": "#8D6E63",
|
||||
"NSC": "#FF8F00",
|
||||
"Nieuw Sociaal Contract": "#FF8F00",
|
||||
"DENK": "#00897B",
|
||||
"50PLUS": "#7E57C2",
|
||||
"Volt": "#572AB7",
|
||||
"ChristenUnie": "#0288D1",
|
||||
"Unknown": "#9E9E9E",
|
||||
}
|
||||
|
||||
SVD_THEMES: dict[int, dict[str, str]] = {
|
||||
1: {
|
||||
"label": "Rechts kabinetsbeleid versus links oppositiebeleid",
|
||||
"explanation": (
|
||||
"Deze as scheidt het rechts kabinetsbeleid van links oppositiebeleid. "
|
||||
"Aan de positieve kant staan moties die passen bij het kabinetsbeleid: "
|
||||
"Eurofighter Typhoons, defensie-uitgaven naar 3% bbp, F-35 reservedelen, "
|
||||
"marine-steun aan Rode Zee en asielrestricties. "
|
||||
"PVV, VVD, NSC en BBB scoren sterk positief. "
|
||||
"Aan de negatieve kant staan moties uit de oppositie: "
|
||||
"zorgbuurthuizen voor ouderen, boycot van Israël, sancties, en internationale "
|
||||
"klimaatsamenwerking. GroenLinks-PvdA, SP, PvdD en Volt scoren negatief. "
|
||||
"Deze as weerspiegelt de coalitie-oppositie dynamiek."
|
||||
),
|
||||
"positive_pole": "Kabinetsbeleid: PVV, VVD, NSC, BBB, JA21 — defensie en restricties",
|
||||
"negative_pole": "Oppositiebeleid: GroenLinks-PvdA, SP, PvdD, Volt, DENK — zorg en multilateraal",
|
||||
"flip": False,
|
||||
},
|
||||
2: {
|
||||
"label": "PVV/FVD-populisme versus mainstream-partijen",
|
||||
"explanation": (
|
||||
"Deze as scheidt het PVV/FVD-populisme van het overige parliament. "
|
||||
"Alleen PVV en FVD scoren positief; alle andere partijen scoren negatief. "
|
||||
"Positieve moties: Syriërs terugsturen, geen geld aan Jordanië, tijdelijke "
|
||||
"bescherming Oekraïne beëindigen, uitstappen uit WHO en klimaatakkoorden. "
|
||||
"Negatieve moties: digitale toegankelijkheid Caribisch Nederland, ethiekprogramma "
|
||||
"Defensie, zorg voor slachtoffers bombardement Hawija, internationale klimaatsamenwerking. "
|
||||
"Dit is geen links-rechts verdeling maar een populistisch vs. mainstream onderscheid."
|
||||
),
|
||||
"positive_pole": "PVV en FVD — soevereiniteit en anti-establishment",
|
||||
"negative_pole": "Overige partijen: VVD, CDA, SGP, ChristenUnie, GroenLinks-PvdA, D66, Volt, BBB",
|
||||
"flip": False,
|
||||
},
|
||||
3: {
|
||||
"label": "Verzorgingsstaat versus bezuinigingen en marktwerking",
|
||||
"explanation": (
|
||||
"Deze as weerspiegelt de spanning tussen staatsingrijpen en marktliberalisme, "
|
||||
"aangescherpt door de kabinetscrisis van 2025. Aan de positieve kant staan moties "
|
||||
"die bezuinigingen op zorg en het gemeentefonds willen terugdraaien, winstuitkeringen "
|
||||
"in de zorg verbieden en publieke controle over ziekenhuisfusies eisen. SP, PvdD, "
|
||||
"GroenLinks-PvdA stemmen hier gelijk — ondanks hun tegengestelde PC1-posities. "
|
||||
"Aan de negatieve kant staan moties "
|
||||
"over marktwerking in de zorg, fiscale bedrijfsopvolgingsfaciliteiten (VVD), "
|
||||
"doorgaan met besturen ondanks de kabinetscrisis (VVD/BBB) en defensie-"
|
||||
"uitgaven van 3,5% bbp."
|
||||
),
|
||||
"positive_pole": "Pro-verzorgingsstaat: SP, PvdD, GroenLinks-PvdA (anti-bezuinigingen)",
|
||||
"negative_pole": "Marktliberaal en fiscaal conservatief: VVD, D66, CDA, SGP, BBB",
|
||||
"flip": True,
|
||||
},
|
||||
4: {
|
||||
"label": "Mainstreampartijen versus FVD/DENK-oppositie",
|
||||
"explanation": (
|
||||
"Deze as scheidt het mainstream parliament van FVD en DENK. "
|
||||
"Aan de positieve kant stemmen vrijwel alle partijen voor dezelfde moties: "
|
||||
"openbare toiletten, vaderbetrokkenheid bij opvoeding, internationale "
|
||||
"samenwerking met Australië en Canada, en long covid-expertise. "
|
||||
"D66, CDA, VVD, PVV, GL-PvdA, SP, Volt en 50PLUS stemmen allemaal samen. "
|
||||
"Aan de negatieve kant stemmen alleen FVD en DENK voor — zij nemen "
|
||||
"regelmatig gepolariseerde posities die afwijken van het mainstream."
|
||||
),
|
||||
"positive_pole": "Mainstreampartijen: D66, CDA, VVD, PVV, GL-PvdA, SP, Volt, 50PLUS — breedgedragen moties",
|
||||
"negative_pole": "FVD en DENK: oppositieposities buiten de mainstream",
|
||||
"flip": True,
|
||||
},
|
||||
5: {
|
||||
"label": "Christelijk-sociaal en gemeenschapswaarden versus progressieve individuele rechten",
|
||||
"explanation": (
|
||||
"Deze as scheidt christelijk-sociale partijen van progressieve partijen op het "
|
||||
"vlak van gemeenschapswaarden. Aan de positieve kant staan moties over "
|
||||
"schuldhulpverlening via vrijwilligersorganisaties, maatschappelijke "
|
||||
"diensttijd voor jongeren, gastouderopvang en financiële prikkels voor scholieren. "
|
||||
"ChristenUnie, SGP, CDA en NSC voeren hier de toon; ook D66 en FVD scoren positief. "
|
||||
"Aan de negatieve kant staan moties over wettelijke erkenning van meerouderschap, "
|
||||
"abortusrecht in het EU-Handvest, armoedebeleid en sociaal-maatschappelijke thema's. "
|
||||
"SP, VVD, GL-PvdA, PvdD en Volt scoren negatief."
|
||||
),
|
||||
"positive_pole": "Christelijk-sociaal: ChristenUnie, SGP, CDA, NSC — gemeenschap en vrijwilligers",
|
||||
"negative_pole": "Progressief-individueel: SP, VVD, GL-PvdA, PvdD, Volt — individuele rechten",
|
||||
"flip": False,
|
||||
},
|
||||
6: {
|
||||
"label": "Migratie en cultuur versus klimaat en progressieve inclusie",
|
||||
"explanation": (
|
||||
"Deze as combineert migratie- en culturele posities. Aan de positieve kant staan "
|
||||
"moties over asielrestricties, nationale cultuur en identiteit, en beperkte "
|
||||
"immigratie. PVV, JA21, BBB, CDA, ChristenUnie, VVD, SGP, FVD en DENK scoren positief. "
|
||||
"Aan de negatieve kant staan moties over klimaatmaatregelen, progressieve "
|
||||
"inclusie, discriminatiebestrijding en internationale samenwerking. "
|
||||
"SP, PvdD, D66, GL-PvdA en Volt scoren negatief. "
|
||||
"De as scheidt partijen met restrictief migratiebeleid van partijen met "
|
||||
"progressief-inclusief beleid."
|
||||
),
|
||||
"positive_pole": "Restrictief migratiebeleid: PVV, JA21, BBB, CDA, ChristenUnie, VVD, SGP, FVD, DENK",
|
||||
"negative_pole": "Progressieve inclusie: SP, PvdD, D66, GL-PvdA, Volt — klimaat en diversiteit",
|
||||
"flip": False,
|
||||
},
|
||||
7: {
|
||||
"label": "Bestuurlijk pragmatisme en implementatie (indicatief)",
|
||||
"explanation": (
|
||||
"Een residuele as die overwegend beleidsdossiers uit 2024 (vorige parlementaire "
|
||||
"periode) omvat. De scores zijn smal (max ~11 punten) en de partijcombinaties "
|
||||
"ideologisch divers — dit label is indicatief. Aan de positieve kant staan "
|
||||
"pragmatische bestuursmoties: een compleet kostenoverzicht van producten van eigen "
|
||||
"bodem, papieren schoolboeken voor basisvaardigheden, een invoeringstoets voor het "
|
||||
"minimumloon en de A2-snelwegplanning. ChristenUnie, Volt, DENK en SP scoren "
|
||||
"positief. Aan de negatieve kant staan meer ideologisch geladen moties: een "
|
||||
"landelijk stookverbod (PvdD), het strafbaar stellen van verbranding van religieuze "
|
||||
"geschriften (DENK), chroom-6 schadevergoedingen en tegenhouden van nieuwe "
|
||||
"gaswinning. GroenLinks-PvdA, VVD, FVD en JA21 scoren negatief."
|
||||
),
|
||||
"positive_pole": "Praktisch-bestuurlijk: ChristenUnie, Volt, SGP, DENK, SP",
|
||||
"negative_pole": "Ideologisch-principieel: GroenLinks-PvdA, VVD, FVD, JA21",
|
||||
"flip": True,
|
||||
},
|
||||
8: {
|
||||
"label": "Vaccinatiebeleid, onderwijs en regionale huisvesting (indicatief)",
|
||||
"explanation": (
|
||||
"Een residuele as die overwegend thematisch diverse moties uit 2024-2025 vangt. "
|
||||
"Aan de positieve kant staan moties over vaccinatiegraad-verlaging voor kinderen, "
|
||||
"een VWO-profiel kunst en cultuur, stages voor mbo-studenten in het buitenland, "
|
||||
"en woningbouw voor jongeren in kleine kernen. BBB, SGP en JA21 scoren positief. "
|
||||
"Aan de negatieve kant staan moties over het instellen van een vaccinatiecommissie, "
|
||||
"heropening van het coronaoversterfte-onderzoek, regionale energiestrategieën "
|
||||
"en toegankelijkheid van het basispakket. SP, DENK en PvdD scoren sterk negatief. "
|
||||
"Deze as combineert onderwijs- en volksgezondheidsposities met regionale "
|
||||
"huisvestingsprioriteiten — het label is indicatief."
|
||||
),
|
||||
"positive_pole": "Onderwijs en volksgezondheid: BBB, SGP, JA21 — vaccinatie, profielkeuze, woningbouw",
|
||||
"negative_pole": "Zorg en toegankelijkheid: SP, DENK, PvdD, Volt — coronaonderzoek, energie, basispakket",
|
||||
"flip": False,
|
||||
},
|
||||
9: {
|
||||
"label": "Pragmatische probleemoplossing versus systeemhervorming (indicatief)",
|
||||
"explanation": (
|
||||
"Deze as scheidt pragmatische, concrete probleemoplossing van idealistische "
|
||||
"systeemhervorming. Aan de positieve kant staan moties over naleving van de "
|
||||
"Financiële-verhoudingswet voor gemeenten, beperking van arbeidsmigratie, "
|
||||
"een nieuwe tandartsopleiding in Rotterdam, een actieplan tegen misbruik van "
|
||||
"hallucinerende geneesmiddelen en oplossingen voor milieuproblemen op Bonaire. "
|
||||
"SGP en ChristenUnie scoren sterk positief; ook DENK en SP. Aan de negatieve kant "
|
||||
"staan moties over een moratorium op geitenstallen, een verbod op gokadvertenties, "
|
||||
"verduidelijking van gronden voor voorlopige hechtenis, een leegstandbelasting "
|
||||
"en end-to-end-encryptie. D66, JA21 en PVV scoren negatief. "
|
||||
"Deze as is indicatief — de scores zijn smal en ideologisch divers."
|
||||
),
|
||||
"positive_pole": "Pragmatisch-bestuurlijk: SGP, ChristenUnie, DENK, SP — concrete oplossingen",
|
||||
"negative_pole": "Systeemhervorming: D66, JA21, PVV — idealistische beleidsposities",
|
||||
"flip": True,
|
||||
},
|
||||
10: {
|
||||
"label": "Kritisch op overheidsbemoeienis versus pro-regulering (indicatief)",
|
||||
"explanation": (
|
||||
"Deze as scheidt partijen die kritisch staan tegenover overheidsbemoeienis van "
|
||||
"partijen die strikte regulering en handhaving steunen. Aan de positieve kant "
|
||||
"staan moties over minder tijdsintensieve schoolinspecties, het recht van "
|
||||
"toeslagenouders op hun persoonlijk dossier, behoud van tegemoetkomingen voor "
|
||||
"arbeidsongeschikten en verlaging van de leeftijdsdrempel voor kindgesprekken. "
|
||||
"DENK, SP en PvdD scoren positief. Aan de negatieve kant staan moties over "
|
||||
"een aangifteplicht voor scholen bij veiligheidsincidenten, een rookverbod in "
|
||||
"auto's met kinderen, braakliggende landbouwgrond en verhoogd beloningsgeld "
|
||||
"voor tipgevers. GroenLinks-PvdA scoort opvallend sterk negatief. "
|
||||
"Deze as is indicatief — de scores zijn smal en de partijcombinaties divers."
|
||||
),
|
||||
"positive_pole": "Kritisch op overheidsbemoeienis: DENK, SP, PvdD — minder inspectielast en lastenverlichting",
|
||||
"negative_pole": "Pro-regulering: GroenLinks-PvdA, CDA, SGP — veiligheid, naleving en handhaving",
|
||||
"flip": True,
|
||||
},
|
||||
}
|
||||
|
||||
KNOWN_MAJOR_PARTIES = [
|
||||
"VVD",
|
||||
"PVV",
|
||||
"D66",
|
||||
"GroenLinks-PvdA",
|
||||
"GroenLinks",
|
||||
"PvdA",
|
||||
"CDA",
|
||||
"SP",
|
||||
"NSC",
|
||||
"CU",
|
||||
"BBB",
|
||||
]
|
||||
|
||||
CURRENT_PARLIAMENT_PARTIES: frozenset[str] = frozenset(
|
||||
{
|
||||
"PVV",
|
||||
"VVD",
|
||||
"NSC",
|
||||
"BBB",
|
||||
"D66",
|
||||
"GroenLinks-PvdA",
|
||||
"CDA",
|
||||
"SP",
|
||||
"ChristenUnie",
|
||||
"SGP",
|
||||
"Volt",
|
||||
"DENK",
|
||||
"PvdD",
|
||||
"JA21",
|
||||
"FVD",
|
||||
}
|
||||
)
|
||||
|
||||
_PARTY_NORMALIZE: dict[str, str] = {
|
||||
"Nieuw Sociaal Contract": "NSC",
|
||||
"CU": "ChristenUnie",
|
||||
"GL": "GroenLinks-PvdA",
|
||||
"GroenLinks": "GroenLinks-PvdA",
|
||||
"PvdA": "GroenLinks-PvdA",
|
||||
"Gündoğan": "Volt",
|
||||
"Lid Keijzer": "BBB",
|
||||
"Groep Markuszower": "PVV",
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
"""Data loading functions for the parliamentary explorer.
|
||||
|
||||
This module contains all data loading functions extracted from explorer.py.
|
||||
It is intentionally free of Streamlit side-effects to be easy to unit test.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Dict, List, Set, Tuple
|
||||
|
||||
import duckdb
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
|
||||
from analysis.config import CURRENT_PARLIAMENT_PARTIES, _PARTY_NORMALIZE
|
||||
|
||||
__all__ = [
|
||||
"get_available_windows",
|
||||
"get_uniform_dim_windows",
|
||||
"load_party_map",
|
||||
"load_active_mps",
|
||||
"load_mp_vectors_by_window",
|
||||
"load_mp_vectors_by_party",
|
||||
"load_mp_vectors_by_party_for_window",
|
||||
"load_party_axis_scores",
|
||||
"load_party_axis_scores_for_window",
|
||||
"load_party_scores_all_windows",
|
||||
"load_party_scores_all_windows_aligned",
|
||||
"load_party_mp_vectors",
|
||||
"build_window_party_scores",
|
||||
"load_motions_df",
|
||||
"query_similar",
|
||||
"compute_party_axis_scores",
|
||||
]
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
_WINDOW_SQL = """
|
||||
SELECT DISTINCT window_id FROM svd_vectors ORDER BY window_id
|
||||
"""
|
||||
|
||||
_UNIFORM_DIM_SQL = """
|
||||
WITH vec_dims AS (
|
||||
SELECT window_id, json_array_length(vector) AS dim
|
||||
FROM svd_vectors
|
||||
WHERE entity_type = 'mp'
|
||||
),
|
||||
window_dim_counts AS (
|
||||
SELECT window_id, dim, COUNT(*) AS cnt
|
||||
FROM vec_dims
|
||||
GROUP BY window_id, dim
|
||||
),
|
||||
dominant AS (
|
||||
SELECT DISTINCT ON (window_id) window_id, dim, cnt
|
||||
FROM window_dim_counts
|
||||
ORDER BY window_id, cnt DESC, dim DESC
|
||||
)
|
||||
SELECT window_id
|
||||
FROM dominant
|
||||
WHERE dim >= 25 AND cnt >= 10
|
||||
ORDER BY window_id
|
||||
"""
|
||||
|
||||
|
||||
def get_available_windows(db_path: str) -> List[str]:
|
||||
"""Return sorted list of distinct window_ids from svd_vectors."""
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
try:
|
||||
rows = con.execute(_WINDOW_SQL).fetchall()
|
||||
return [r[0] for r in rows]
|
||||
except Exception:
|
||||
logger.exception("Failed to query available windows")
|
||||
return []
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def get_uniform_dim_windows(db_path: str) -> List[str]:
|
||||
"""Return only windows whose dominant MP-vector dimension is >= 25.
|
||||
|
||||
Some windows contain a mix of vector lengths due to multiple pipeline runs
|
||||
(e.g. 2016 has both dim=1 and dim=50 rows). We find the most common dimension
|
||||
per window and include only windows where that dominant dim >= 25.
|
||||
Windows with too few dim-25+ entities (< 10) are also excluded to avoid
|
||||
degenerate PCA inputs.
|
||||
"""
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
try:
|
||||
rows = con.execute(_UNIFORM_DIM_SQL).fetchall()
|
||||
return [r[0] for r in rows]
|
||||
except Exception:
|
||||
logger.exception("Failed to query uniform-dim windows")
|
||||
return []
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def load_party_map(db_path: str) -> Dict[str, str]:
|
||||
"""Return {mp_name: party} mapping, with party names normalised to abbreviations."""
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
rows = con.execute(
|
||||
"SELECT mp_name, party FROM mp_metadata WHERE party IS NOT NULL"
|
||||
).fetchall()
|
||||
con.close()
|
||||
return {
|
||||
mp: _PARTY_NORMALIZE.get(party, party) for mp, party in rows if mp and party
|
||||
}
|
||||
except Exception:
|
||||
logger.exception("Failed to load party map")
|
||||
return {}
|
||||
|
||||
|
||||
def load_active_mps(db_path: str) -> Set[str]:
|
||||
"""Return the set of mp_name values that are currently seated in parliament.
|
||||
|
||||
An MP is considered active if their mp_metadata row has tot_en_met IS NULL,
|
||||
meaning they have no recorded end date for their current seat.
|
||||
"""
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
rows = con.execute(
|
||||
"SELECT mp_name FROM mp_metadata WHERE tot_en_met IS NULL"
|
||||
).fetchall()
|
||||
con.close()
|
||||
return {r[0] for r in rows if r[0]}
|
||||
except Exception:
|
||||
logger.exception("Failed to load active MPs")
|
||||
return set()
|
||||
|
||||
|
||||
def load_party_axis_scores(db_path: str) -> Dict[str, List[float]]:
|
||||
"""Return party scores for all windows (non-aligned).
|
||||
|
||||
Returns dict mapping party_abbrev -> list of axis scores, one per window.
|
||||
"""
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT party_abbrev, window_id, x_axis, y_axis
|
||||
FROM party_axis_scores
|
||||
ORDER BY party_abbrev, window_id
|
||||
"""
|
||||
).fetchall()
|
||||
con.close()
|
||||
|
||||
scores: Dict[str, List[float]] = {}
|
||||
for party, window, x, y in rows:
|
||||
if party not in scores:
|
||||
scores[party] = []
|
||||
if x is not None and y is not None:
|
||||
scores[party].extend([x, y])
|
||||
return scores
|
||||
except Exception:
|
||||
logger.exception("Failed to load party axis scores")
|
||||
return {}
|
||||
|
||||
|
||||
def load_party_axis_scores_for_window(
|
||||
db_path: str, window: str
|
||||
) -> Dict[str, List[float]]:
|
||||
"""Return party scores for a specific window (aligned)."""
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT party_abbrev, x_axis, y_axis
|
||||
FROM party_axis_scores
|
||||
WHERE window_id = ?
|
||||
ORDER BY party_abbrev
|
||||
""",
|
||||
[window],
|
||||
).fetchall()
|
||||
con.close()
|
||||
|
||||
return {party: [x or 0.0, y or 0.0] for party, x, y in rows}
|
||||
except Exception:
|
||||
logger.exception("Failed to load party axis scores for window %s", window)
|
||||
return {}
|
||||
|
||||
|
||||
def load_party_scores_all_windows(db_path: str) -> Dict[str, List[List[float]]]:
|
||||
"""Return party scores across all windows (non-aligned)."""
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT party_abbrev, window_id, x_axis, y_axis
|
||||
FROM party_axis_scores
|
||||
ORDER BY party_abbrev, window_id
|
||||
"""
|
||||
).fetchall()
|
||||
con.close()
|
||||
|
||||
scores: Dict[str, List[List[float]]] = {}
|
||||
current_party = None
|
||||
for party, window, x, y in rows:
|
||||
if party != current_party:
|
||||
scores[party] = []
|
||||
current_party = party
|
||||
if x is not None and y is not None:
|
||||
scores[party].append([x, y])
|
||||
else:
|
||||
scores[party].append([0.0, 0.0])
|
||||
return scores
|
||||
except Exception:
|
||||
logger.exception("Failed to load party scores all windows")
|
||||
return {}
|
||||
|
||||
|
||||
def load_party_scores_all_windows_aligned(
|
||||
db_path: str,
|
||||
) -> Dict[str, List[List[float]]]:
|
||||
"""Return party scores across all windows (Procrustes-aligned)."""
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT party_abbrev, window_id, x_axis_aligned, y_axis_aligned
|
||||
FROM party_axis_scores
|
||||
ORDER BY party_abbrev, window_id
|
||||
"""
|
||||
).fetchall()
|
||||
con.close()
|
||||
|
||||
scores: Dict[str, List[List[float]]] = {}
|
||||
current_party = None
|
||||
for party, window, x, y in rows:
|
||||
if party != current_party:
|
||||
scores[party] = []
|
||||
current_party = party
|
||||
if x is not None and y is not None:
|
||||
scores[party].append([x, y])
|
||||
else:
|
||||
scores[party].append([0.0, 0.0])
|
||||
return scores
|
||||
except Exception:
|
||||
logger.exception("Failed to load aligned party scores all windows")
|
||||
return {}
|
||||
|
||||
|
||||
def build_window_party_scores(
|
||||
scores_by_party: Dict[str, List[List[float]]],
|
||||
window_idx: int,
|
||||
) -> Dict[str, List[float]]:
|
||||
"""Extract scores for one window as {party: [x, y]} for compute_flip_direction.
|
||||
|
||||
Args:
|
||||
scores_by_party: Output of load_party_scores_all_windows_aligned —
|
||||
{party: [[x, y], [x, y], ...]} per window.
|
||||
window_idx: Zero-based index of the window to extract.
|
||||
|
||||
Returns:
|
||||
{party: [x, y]} for the given window. Returns empty dict if
|
||||
window_idx is out of range.
|
||||
"""
|
||||
if window_idx < 0:
|
||||
return {}
|
||||
result: Dict[str, List[float]] = {}
|
||||
for party, window_scores in scores_by_party.items():
|
||||
if window_idx < len(window_scores):
|
||||
result[party] = window_scores[window_idx]
|
||||
return result
|
||||
|
||||
|
||||
def load_party_mp_vectors(db_path: str) -> Dict[str, List[np.ndarray]]:
|
||||
"""Load individual MP SVD vectors grouped by party.
|
||||
|
||||
Returns {party_name: [np.ndarray(50,), ...]} — one array per MP.
|
||||
"""
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
try:
|
||||
meta_rows = con.execute(
|
||||
"SELECT mp_name, party FROM mp_metadata "
|
||||
"WHERE van >= '2023-11-22' OR tot_en_met IS NULL OR tot_en_met >= '2023-11-22' "
|
||||
"ORDER BY van ASC"
|
||||
).fetchall()
|
||||
mp_party: Dict[str, str] = {}
|
||||
for mp_name, party in meta_rows:
|
||||
if mp_name and party:
|
||||
mp_party[mp_name] = _PARTY_NORMALIZE.get(party, party)
|
||||
|
||||
rows = con.execute(
|
||||
"SELECT entity_id, vector FROM svd_vectors "
|
||||
"WHERE entity_type = 'mp' AND window_id = 'current_parliament'"
|
||||
).fetchall()
|
||||
|
||||
vectors_by_party: Dict[str, List[np.ndarray]] = {}
|
||||
for entity_id, vector_json in rows:
|
||||
if entity_id in mp_party:
|
||||
party = mp_party[entity_id]
|
||||
if party not in vectors_by_party:
|
||||
vectors_by_party[party] = []
|
||||
vectors_by_party[party].append(np.array(vector_json))
|
||||
|
||||
return vectors_by_party
|
||||
except Exception:
|
||||
logger.exception("Failed to load party MP vectors")
|
||||
return {}
|
||||
finally:
|
||||
con.close()
|
||||
|
||||
|
||||
def load_scree_data(db_path: str) -> List[float]:
|
||||
"""Load scree plot data (explained variance) for current_parliament."""
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
row = con.execute(
|
||||
"""
|
||||
SELECT sv_metadata FROM svd_vectors
|
||||
WHERE window_id = 'current_parliament' AND entity_type = 'singular_values'
|
||||
LIMIT 1
|
||||
"""
|
||||
).fetchone()
|
||||
con.close()
|
||||
|
||||
if row and row[0]:
|
||||
import json
|
||||
|
||||
return json.loads(row[0])
|
||||
return []
|
||||
except Exception:
|
||||
logger.exception("Failed to load scree data")
|
||||
return []
|
||||
|
||||
|
||||
def load_motions_df(db_path: str) -> pd.DataFrame:
|
||||
"""Load the full motions table as a pandas DataFrame (read-only)."""
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
df = con.execute(
|
||||
"""
|
||||
SELECT id, title, description, date, policy_area,
|
||||
voting_results, layman_explanation,
|
||||
winning_margin, controversy_score, url
|
||||
FROM motions
|
||||
"""
|
||||
).fetchdf()
|
||||
con.close()
|
||||
df["date"] = pd.to_datetime(df["date"], errors="coerce")
|
||||
df["year"] = df["date"].dt.year
|
||||
return df
|
||||
except Exception:
|
||||
logger.exception("Failed to load motions DataFrame")
|
||||
return pd.DataFrame()
|
||||
|
||||
|
||||
def load_mp_vectors_by_window(db_path: str, window: str) -> Dict[str, np.ndarray]:
|
||||
"""Load individual MP SVD vectors for a specific window.
|
||||
|
||||
Args:
|
||||
db_path: Path to DuckDB database
|
||||
window: Window ID (e.g., "2015", "current_parliament")
|
||||
|
||||
Returns:
|
||||
{mp_name: np.ndarray(50,)} — one vector per MP
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT entity_id, vector FROM svd_vectors
|
||||
WHERE entity_type = 'mp' AND window_id = ?
|
||||
""",
|
||||
[window],
|
||||
).fetchall()
|
||||
con.close()
|
||||
|
||||
mp_vecs: Dict[str, np.ndarray] = {}
|
||||
for entity_id, raw_vec in rows:
|
||||
if isinstance(raw_vec, str):
|
||||
vec = _json.loads(raw_vec)
|
||||
elif isinstance(raw_vec, (bytes, bytearray)):
|
||||
vec = _json.loads(raw_vec.decode())
|
||||
elif isinstance(raw_vec, list):
|
||||
vec = raw_vec
|
||||
else:
|
||||
try:
|
||||
vec = list(raw_vec)
|
||||
except Exception:
|
||||
continue
|
||||
fvec = np.array([float(v) if v is not None else 0.0 for v in vec])
|
||||
mp_vecs[entity_id] = fvec
|
||||
|
||||
return mp_vecs
|
||||
except Exception:
|
||||
logger.exception("Failed to load MP vectors for window %s", window)
|
||||
return {}
|
||||
|
||||
|
||||
def query_similar(
|
||||
db_path: str,
|
||||
source_motion_id: int,
|
||||
vector_type: str = "fused",
|
||||
top_k: int = 10,
|
||||
) -> pd.DataFrame:
|
||||
"""Return top-k similar motions from similarity_cache (read-only)."""
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
rows = con.execute(
|
||||
"""
|
||||
SELECT sc.target_motion_id, sc.score, sc.window_id,
|
||||
m.title, m.date, m.policy_area
|
||||
FROM similarity_cache sc
|
||||
JOIN motions m ON m.id = sc.target_motion_id
|
||||
WHERE sc.source_motion_id = ?
|
||||
AND sc.vector_type = ?
|
||||
ORDER BY sc.score DESC
|
||||
LIMIT ?
|
||||
""",
|
||||
[source_motion_id, vector_type, top_k],
|
||||
).fetchdf()
|
||||
con.close()
|
||||
return rows
|
||||
except Exception:
|
||||
logger.exception(
|
||||
"Failed to query similarity cache for motion %s", source_motion_id
|
||||
)
|
||||
return pd.DataFrame()
|
||||
|
||||
|
||||
def load_mp_vectors_by_party(db_path: str) -> Dict[str, List[np.ndarray]]:
|
||||
"""Load individual MP SVD vectors grouped by party for current_parliament.
|
||||
|
||||
Returns:
|
||||
{party_name: [np.ndarray(50,), ...]} — one array per MP.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
meta_rows = con.execute(
|
||||
"SELECT mp_name, party FROM mp_metadata "
|
||||
"WHERE van >= '2023-11-22' OR tot_en_met IS NULL OR tot_en_met >= '2023-11-22' "
|
||||
"ORDER BY van ASC"
|
||||
).fetchall()
|
||||
mp_party: Dict[str, str] = {}
|
||||
for mp_name, party in meta_rows:
|
||||
if mp_name and party:
|
||||
mp_party[mp_name] = _PARTY_NORMALIZE.get(party, party)
|
||||
|
||||
rows = con.execute(
|
||||
"SELECT entity_id, vector FROM svd_vectors "
|
||||
"WHERE entity_type='mp' AND window_id='current_parliament'"
|
||||
).fetchall()
|
||||
con.close()
|
||||
|
||||
party_vecs: Dict[str, List[np.ndarray]] = {}
|
||||
for entity_id, raw_vec in rows:
|
||||
party = mp_party.get(entity_id)
|
||||
if party is None or party not in CURRENT_PARLIAMENT_PARTIES:
|
||||
continue
|
||||
if isinstance(raw_vec, str):
|
||||
vec = _json.loads(raw_vec)
|
||||
elif isinstance(raw_vec, (bytes, bytearray)):
|
||||
vec = _json.loads(raw_vec.decode())
|
||||
elif isinstance(raw_vec, list):
|
||||
vec = raw_vec
|
||||
else:
|
||||
try:
|
||||
vec = list(raw_vec)
|
||||
except Exception:
|
||||
continue
|
||||
fvec = np.array([float(v) if v is not None else 0.0 for v in vec])
|
||||
party_vecs.setdefault(party, []).append(fvec)
|
||||
return party_vecs
|
||||
except Exception:
|
||||
logger.exception("Failed to load MP vectors by party")
|
||||
return {}
|
||||
|
||||
|
||||
def load_mp_vectors_by_party_for_window(
|
||||
db_path: str, window: str
|
||||
) -> Dict[str, List[np.ndarray]]:
|
||||
"""Load individual MP SVD vectors grouped by party for a specific window.
|
||||
|
||||
For historical windows, uses the MP→party mapping from that time period.
|
||||
|
||||
Returns:
|
||||
{party_name: [np.ndarray(50,), ...]} — one array per MP.
|
||||
"""
|
||||
import json as _json
|
||||
|
||||
try:
|
||||
con = duckdb.connect(database=db_path, read_only=True)
|
||||
is_current = window == "current_parliament"
|
||||
|
||||
if is_current:
|
||||
meta_rows = con.execute(
|
||||
"SELECT mp_name, party FROM mp_metadata "
|
||||
"WHERE van >= '2023-11-22' OR tot_en_met IS NULL OR tot_en_met >= '2023-11-22' "
|
||||
"ORDER BY van ASC"
|
||||
).fetchall()
|
||||
else:
|
||||
try:
|
||||
year = int(window.split("-")[0])
|
||||
except ValueError:
|
||||
year = 2023
|
||||
meta_rows = con.execute(
|
||||
"SELECT mp_name, party FROM mp_metadata "
|
||||
"WHERE van <= ? AND (tot_en_met IS NULL OR tot_en_met >= ?) "
|
||||
"ORDER BY van ASC",
|
||||
[f"{year}-12-31", f"{year}-01-01"],
|
||||
).fetchall()
|
||||
|
||||
mp_party: Dict[str, str] = {}
|
||||
for mp_name, party in meta_rows:
|
||||
if mp_name and party:
|
||||
mp_party[mp_name] = _PARTY_NORMALIZE.get(party, party)
|
||||
|
||||
rows = con.execute(
|
||||
"SELECT entity_id, vector FROM svd_vectors "
|
||||
"WHERE entity_type='mp' AND window_id=?",
|
||||
[window],
|
||||
).fetchall()
|
||||
con.close()
|
||||
|
||||
party_vecs: Dict[str, List[np.ndarray]] = {}
|
||||
for entity_id, raw_vec in rows:
|
||||
party = mp_party.get(entity_id)
|
||||
if party is None:
|
||||
continue
|
||||
if is_current and party not in CURRENT_PARLIAMENT_PARTIES:
|
||||
continue
|
||||
if isinstance(raw_vec, str):
|
||||
vec = _json.loads(raw_vec)
|
||||
elif isinstance(raw_vec, (bytes, bytearray)):
|
||||
vec = _json.loads(raw_vec.decode())
|
||||
elif isinstance(raw_vec, list):
|
||||
vec = raw_vec
|
||||
else:
|
||||
try:
|
||||
vec = list(raw_vec)
|
||||
except Exception:
|
||||
continue
|
||||
fvec = np.array([float(v) if v is not None else 0.0 for v in vec])
|
||||
party_vecs.setdefault(party, []).append(fvec)
|
||||
return party_vecs
|
||||
except Exception:
|
||||
logger.exception("Failed to load MP vectors by party for window %s", window)
|
||||
return {}
|
||||
|
||||
|
||||
def compute_party_axis_scores(
|
||||
party_vecs: Dict[str, List[np.ndarray]],
|
||||
) -> Dict[str, List[float]]:
|
||||
"""Compute per-party axis scores as mean of MP vectors.
|
||||
|
||||
Returns:
|
||||
{party_name: [float * k]} — k = 50, mean over all MPs in that party.
|
||||
"""
|
||||
try:
|
||||
return {
|
||||
party: np.array(vecs).mean(axis=0).tolist()
|
||||
for party, vecs in party_vecs.items()
|
||||
}
|
||||
except Exception:
|
||||
logger.exception("Failed to compute party axis scores")
|
||||
return {}
|
||||
+9
-28
@@ -8,33 +8,12 @@ directions automatically based on party centroids.
|
||||
import logging
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
|
||||
from analysis.config import CANONICAL_LEFT, CANONICAL_RIGHT
|
||||
|
||||
_logger = logging.getLogger(__name__)
|
||||
|
||||
# Canonical party sets for orientation
|
||||
# Right-wing parties that should appear on the right side of axes
|
||||
RIGHT_PARTIES = {
|
||||
"PVV",
|
||||
"VVD",
|
||||
"FVD",
|
||||
"BBB",
|
||||
"JA21",
|
||||
"Nieuw Sociaal Contract",
|
||||
"SGP",
|
||||
"CDA",
|
||||
"ChristenUnie",
|
||||
}
|
||||
|
||||
# Left-wing parties that should appear on the left side of axes
|
||||
LEFT_PARTIES = {
|
||||
"SP",
|
||||
"PvdA",
|
||||
"GL",
|
||||
"GroenLinks",
|
||||
"GroenLinks-PvdA",
|
||||
"DENK",
|
||||
"PvdD",
|
||||
"Volt",
|
||||
}
|
||||
RIGHT_PARTIES = CANONICAL_RIGHT
|
||||
LEFT_PARTIES = CANONICAL_LEFT
|
||||
|
||||
# Cache for SVD_THEMES to avoid repeated imports
|
||||
_svd_themes_cache: Optional[Dict[int, Dict[str, str]]] = None
|
||||
@@ -125,14 +104,16 @@ def get_svd_theme(component: int) -> Dict[str, str]:
|
||||
|
||||
|
||||
def compute_flip_direction(
|
||||
component: int, party_scores: Dict[str, List[float]]
|
||||
component: int,
|
||||
party_scores: Dict[str, List[float]],
|
||||
) -> bool:
|
||||
"""Compute flip direction so right parties appear on the right side.
|
||||
|
||||
Args:
|
||||
component: SVD component number (1-indexed)
|
||||
party_scores: Dict mapping party name to list of scores per component
|
||||
(party_scores[party][0] is score for component 1, etc.)
|
||||
party_scores: Dict mapping party name to per-component scores.
|
||||
party_scores[party][0] is score for component 1 (x-axis),
|
||||
party_scores[party][1] is score for component 2 (y-axis).
|
||||
|
||||
Returns:
|
||||
True if axis should be flipped so right parties are on right.
|
||||
|
||||
Reference in New Issue
Block a user