fix: resolve SVD axis label alignment and score mismatch in tijdtraject view

Two related bugs fixed:

1. Label alignment: Removed static left_pole/right_pole from SVD_THEMES
   entries. These labels assumed a fixed flip direction but could mismatch
   with runtime flip computation, causing right-wing parties to appear on
   the wrong side. Labels are now always derived from positive_pole,
   negative_pole, and the runtime flip direction.

2. Score mismatch: Changed tijdtraject view for components 3-10 from
   load_party_scores_all_windows_aligned() to load_party_scores_all_windows().
   Procrustes alignment rotates the full 50-dim vector space to align
   components 1-2, but this also transforms components 3-10, making their
   scores incomparable with the single-window view. Per-window flip
   computation already handles orientation alignment for these components.

Also updated svd_labels.py to prefer analysis.config as the canonical
source for SVD_THEMES, falling back to explorer only when config is
unavailable.
main
Sven Geboers 3 weeks ago
parent 54489b6a30
commit 823df6f9ee
  1. 20
      analysis/config.py
  2. 22
      analysis/svd_labels.py
  3. 25
      explorer.py
  4. 13
      scripts/validate_svd_themes.py

@ -80,8 +80,6 @@ SVD_THEMES: dict[int, dict[str, str]] = {
), ),
"positive_pole": "Rechts: PVV, VVD, NSC, BBB, JA21 — kabinetsbeleid, defensie en restricties", "positive_pole": "Rechts: PVV, VVD, NSC, BBB, JA21 — kabinetsbeleid, defensie en restricties",
"negative_pole": "Links: GroenLinks-PvdA, SP, PvdD, Volt, DENK — oppositie, zorg en multilateraal", "negative_pole": "Links: GroenLinks-PvdA, SP, PvdD, Volt, DENK — oppositie, zorg en multilateraal",
"left_pole": "Links: GroenLinks-PvdA, SP, PvdD, Volt, DENK — oppositie, zorg en multilateraal",
"right_pole": "Rechts: PVV, VVD, NSC, BBB, JA21 — kabinetsbeleid, defensie en restricties",
"flip": False, "flip": False,
}, },
2: { 2: {
@ -97,8 +95,6 @@ SVD_THEMES: dict[int, dict[str, str]] = {
), ),
"positive_pole": "PVV en FVD — soevereiniteit en anti-establishment", "positive_pole": "PVV en FVD — soevereiniteit en anti-establishment",
"negative_pole": "Overige partijen: VVD, CDA, SGP, ChristenUnie, GroenLinks-PvdA, D66, Volt, BBB", "negative_pole": "Overige partijen: VVD, CDA, SGP, ChristenUnie, GroenLinks-PvdA, D66, Volt, BBB",
"left_pole": "Overige partijen: VVD, CDA, SGP, ChristenUnie, GroenLinks-PvdA, D66, Volt, BBB",
"right_pole": "PVV en FVD — soevereiniteit en anti-establishment",
"flip": False, "flip": False,
}, },
3: { 3: {
@ -116,8 +112,6 @@ SVD_THEMES: dict[int, dict[str, str]] = {
), ),
"positive_pole": "Pro-verzorgingsstaat: SP, PvdD, GroenLinks-PvdA (anti-bezuinigingen)", "positive_pole": "Pro-verzorgingsstaat: SP, PvdD, GroenLinks-PvdA (anti-bezuinigingen)",
"negative_pole": "Marktliberaal en fiscaal conservatief: VVD, D66, CDA, SGP, BBB", "negative_pole": "Marktliberaal en fiscaal conservatief: VVD, D66, CDA, SGP, BBB",
"left_pole": "Marktliberaal en fiscaal conservatief: VVD, D66, CDA, SGP, BBB",
"right_pole": "Pro-verzorgingsstaat: SP, PvdD, GroenLinks-PvdA (anti-bezuinigingen)",
"flip": True, "flip": True,
}, },
4: { 4: {
@ -132,8 +126,6 @@ SVD_THEMES: dict[int, dict[str, str]] = {
), ),
"positive_pole": "D66, CDA, JA21 — moties met brede steun", "positive_pole": "D66, CDA, JA21 — moties met brede steun",
"negative_pole": "NSC, BBB — moties met andere focus", "negative_pole": "NSC, BBB — moties met andere focus",
"left_pole": "NSC, BBB — moties met andere focus",
"right_pole": "D66, CDA, JA21 — moties met brede steun",
"flip": True, "flip": True,
}, },
5: { 5: {
@ -150,8 +142,6 @@ SVD_THEMES: dict[int, dict[str, str]] = {
), ),
"positive_pole": "Christelijk-sociaal: ChristenUnie, SGP, CDA, NSC — gemeenschap en vrijwilligers", "positive_pole": "Christelijk-sociaal: ChristenUnie, SGP, CDA, NSC — gemeenschap en vrijwilligers",
"negative_pole": "Progressief-individueel: SP, VVD, GL-PvdA, PvdD, Volt — individuele rechten", "negative_pole": "Progressief-individueel: SP, VVD, GL-PvdA, PvdD, Volt — individuele rechten",
"left_pole": "Progressief-individueel: SP, VVD, GL-PvdA, PvdD, Volt — individuele rechten",
"right_pole": "Christelijk-sociaal: ChristenUnie, SGP, CDA, NSC — gemeenschap en vrijwilligers",
"flip": False, "flip": False,
}, },
6: { 6: {
@ -168,8 +158,6 @@ SVD_THEMES: dict[int, dict[str, str]] = {
), ),
"positive_pole": "Restrictief migratiebeleid: PVV, JA21, BBB, CDA, ChristenUnie, VVD, SGP, FVD, DENK", "positive_pole": "Restrictief migratiebeleid: PVV, JA21, BBB, CDA, ChristenUnie, VVD, SGP, FVD, DENK",
"negative_pole": "Progressieve inclusie: SP, PvdD, D66, GL-PvdA, Volt — klimaat en diversiteit", "negative_pole": "Progressieve inclusie: SP, PvdD, D66, GL-PvdA, Volt — klimaat en diversiteit",
"left_pole": "Progressieve inclusie: SP, PvdD, D66, GL-PvdA, Volt — klimaat en diversiteit",
"right_pole": "Restrictief migratiebeleid: PVV, JA21, BBB, CDA, ChristenUnie, VVD, SGP, FVD, DENK",
"flip": False, "flip": False,
}, },
7: { 7: {
@ -188,8 +176,6 @@ SVD_THEMES: dict[int, dict[str, str]] = {
), ),
"positive_pole": "Praktisch-bestuurlijk: ChristenUnie, Volt, SGP, DENK, SP", "positive_pole": "Praktisch-bestuurlijk: ChristenUnie, Volt, SGP, DENK, SP",
"negative_pole": "Ideologisch-principieel: GroenLinks-PvdA, VVD, FVD, JA21", "negative_pole": "Ideologisch-principieel: GroenLinks-PvdA, VVD, FVD, JA21",
"left_pole": "Ideologisch-principieel: GroenLinks-PvdA, VVD, FVD, JA21",
"right_pole": "Praktisch-bestuurlijk: ChristenUnie, Volt, SGP, DENK, SP",
"flip": True, "flip": True,
}, },
8: { 8: {
@ -207,8 +193,6 @@ SVD_THEMES: dict[int, dict[str, str]] = {
), ),
"positive_pole": "Onderwijs en volksgezondheid: BBB, SGP, JA21 — vaccinatie, profielkeuze, woningbouw", "positive_pole": "Onderwijs en volksgezondheid: BBB, SGP, JA21 — vaccinatie, profielkeuze, woningbouw",
"negative_pole": "Zorg en toegankelijkheid: SP, DENK, PvdD, Volt — coronaonderzoek, energie, basispakket", "negative_pole": "Zorg en toegankelijkheid: SP, DENK, PvdD, Volt — coronaonderzoek, energie, basispakket",
"left_pole": "Zorg en toegankelijkheid: SP, DENK, PvdD, Volt — coronaonderzoek, energie, basispakket",
"right_pole": "Onderwijs en volksgezondheid: BBB, SGP, JA21 — vaccinatie, profielkeuze, woningbouw",
"flip": False, "flip": False,
}, },
9: { 9: {
@ -227,8 +211,6 @@ SVD_THEMES: dict[int, dict[str, str]] = {
), ),
"positive_pole": "Pragmatisch-bestuurlijk: SGP, ChristenUnie, DENK, SP — concrete oplossingen", "positive_pole": "Pragmatisch-bestuurlijk: SGP, ChristenUnie, DENK, SP — concrete oplossingen",
"negative_pole": "Systeemhervorming: D66, JA21, PVV — idealistische beleidsposities", "negative_pole": "Systeemhervorming: D66, JA21, PVV — idealistische beleidsposities",
"left_pole": "Systeemhervorming: D66, JA21, PVV — idealistische beleidsposities",
"right_pole": "Pragmatisch-bestuurlijk: SGP, ChristenUnie, DENK, SP — concrete oplossingen",
"flip": True, "flip": True,
}, },
10: { 10: {
@ -247,8 +229,6 @@ SVD_THEMES: dict[int, dict[str, str]] = {
), ),
"positive_pole": "Kritisch op overheidsbemoeienis: DENK, SP, PvdD — minder inspectielast en lastenverlichting", "positive_pole": "Kritisch op overheidsbemoeienis: DENK, SP, PvdD — minder inspectielast en lastenverlichting",
"negative_pole": "Pro-regulering: GroenLinks-PvdA, CDA, SGP — veiligheid, naleving en handhaving", "negative_pole": "Pro-regulering: GroenLinks-PvdA, CDA, SGP — veiligheid, naleving en handhaving",
"left_pole": "Pro-regulering: GroenLinks-PvdA, CDA, SGP — veiligheid, naleving en handhaving",
"right_pole": "Kritisch op overheidsbemoeienis: DENK, SP, PvdD — minder inspectielast en lastenverlichting",
"flip": True, "flip": True,
}, },
} }

@ -34,12 +34,28 @@ def _get_svd_themes() -> Dict[int, Dict[str, str]]:
if _svd_themes_cache is not None: if _svd_themes_cache is not None:
return _svd_themes_cache return _svd_themes_cache
# Prefer the lightweight canonical source in analysis.config which is
# intentionally free of heavy runtime dependencies. Fall back to
# explorer.SVD_THEMES only when the config module is unavailable or
# doesn't expose SVD_THEMES.
try: try:
# Import SVD_THEMES from explorer at runtime to avoid circular imports from analysis import config as _cfg
# explorer.py now exports SVD_THEMES at module level
_svd_themes_cache = getattr(_cfg, "SVD_THEMES", {}) or {}
if _svd_themes_cache:
return _svd_themes_cache
except Exception:
_logger.exception(
"Could not import analysis.config or read SVD_THEMES; falling back to explorer"
)
try:
# Import explorer at runtime as a last resort; explorer may pull in
# heavy dependencies (duckdb/plotly) so we only try this if config
# didn't provide the themes.
import explorer import explorer
_svd_themes_cache = explorer.SVD_THEMES _svd_themes_cache = getattr(explorer, "SVD_THEMES", {}) or {}
return _svd_themes_cache return _svd_themes_cache
except ImportError as e: except ImportError as e:
_logger.warning("Could not import explorer.SVD_THEMES: %s", e) _logger.warning("Could not import explorer.SVD_THEMES: %s", e)

@ -2741,10 +2741,13 @@ def build_svd_components_tab(db_path: str) -> None:
party_scores[party] = [avg_x, avg_y] party_scores[party] = [avg_x, avg_y]
party_scores_by_window[window] = party_scores party_scores_by_window[window] = party_scores
else: else:
# Use SVD vectors for components 3-10 with Procrustes alignment # Use raw (non-aligned) SVD vectors for components 3-10.
party_scores_by_window = load_party_scores_all_windows_aligned( # Procrustes alignment rotates the full vector space to align
db_path, all_windows # components 1-2 across windows, but this also transforms
) # components 3-10, making their scores incomparable with the
# single-window view. Per-window flip computation handles
# orientation alignment for these components.
party_scores_by_window = load_party_scores_all_windows(db_path, all_windows)
_render_svd_time_trajectory( _render_svd_time_trajectory(
party_scores_by_window, party_scores_by_window,
@ -2806,16 +2809,10 @@ def build_svd_components_tab(db_path: str) -> None:
pos_pole = theme.get("positive_pole", "") if theme else "" pos_pole = theme.get("positive_pole", "") if theme else ""
neg_pole = theme.get("negative_pole", "") if theme else "" neg_pole = theme.get("negative_pole", "") if theme else ""
# Use semantic left/right pole labels if available, otherwise compute from flip # Derive left/right labels from flip direction
semantic_left = theme.get("left_pole") if theme else None # flip=True: positive_pole on left, negative_pole on right
semantic_right = theme.get("right_pole") if theme else None # flip=False: negative_pole on left, positive_pole on right
if semantic_left and semantic_right: if flip:
left_pole, right_pole = semantic_left, semantic_right
left_motions, right_motions = (
(pos_motions, neg_motions) if flip else (neg_motions, pos_motions)
)
left_arrow, right_arrow = ("", "") if flip else ("", "")
elif flip:
left_pole, right_pole = pos_pole, neg_pole left_pole, right_pole = pos_pole, neg_pole
left_motions, right_motions = pos_motions, neg_motions left_motions, right_motions = pos_motions, neg_motions
left_arrow, right_arrow = "", "" left_arrow, right_arrow = "", ""

@ -201,9 +201,8 @@ def check_theme_consistency(
) -> List[Dict]: ) -> List[Dict]:
"""Check that theme pole labels are consistent with actual party positions. """Check that theme pole labels are consistent with actual party positions.
Note: left_pole/right_pole describe the SEMANTIC left/right after flip, Note: positive_pole/negative_pole describe the SVD axis orientation.
not the political left/right spectrum. This check verifies that the Labels are derived at runtime from flip direction.
parties mentioned in each pole are actually on the expected side.
Returns list of divergence reports. Returns list of divergence reports.
""" """
@ -276,13 +275,7 @@ def main() -> int:
print(f" Diff (post-flip R - L): {d['diff']:.4f}") print(f" Diff (post-flip R - L): {d['diff']:.4f}")
print(f" Right scores: {d['right_scores']}") print(f" Right scores: {d['right_scores']}")
print(f" Left scores: {d['left_scores']}") print(f" Left scores: {d['left_scores']}")
elif d["issue"] == "theme_pole_mismatch":
print(f" Label: {d.get('label', '')}")
print(f" Left pole: {d['left_pole']}")
print(f" Right pole: {d['right_pole']}")
print(f" Left mean: {d['left_mean']:.4f} ({d['left_parties']})")
print(f" Right mean: {d['right_mean']:.4f} ({d['right_parties']})")
print(f" Diff (left - right): {d['diff']:.4f}")
elif d["issue"] == "missing_canonical_party_data": elif d["issue"] == "missing_canonical_party_data":
print(f" Expected right: {canonical_right}") print(f" Expected right: {canonical_right}")
print(f" Expected left: {canonical_left}") print(f" Expected left: {canonical_left}")

Loading…
Cancel
Save