refactor: extract shared helpers to common.py, fix bugs, add TDD tests
- Created analysis/right_wing/common.py with all shared helpers: Constants: CANONICAL_CENTRIST, COALITION, BREAK_YEAR, etc. Functions: _conn, cohens_d, build_party_name_map, parse_lead_submitter, motion_passed, quarter_sort_key, find_inflection_point - Fixed bugs: 1. ai_provider.py: requests.Timeout now caught alongside ConnectionError 2. voting_margin.py: Removed walrus operator misuse, fixed Mann-Whitney test - Updated 13 consuming files to import from common.py - Added 35 TDD tests in tests/right_wing/test_common.py - 282 tests pass (was 247)
This commit is contained in:
@@ -30,27 +30,12 @@ import numpy as np
|
||||
ROOT = Path(__file__).parent.parent.parent.resolve()
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
DB_PATH = str(ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = ROOT / "reports" / "overton_window"
|
||||
from analysis.right_wing.common import (
|
||||
CANONICAL_CENTRIST, COALITION, DB_PATH, REPORTS_DIR,
|
||||
build_party_name_map, parse_lead_submitter, quarter_sort_key,
|
||||
)
|
||||
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
CANONICAL_RIGHT = frozenset({"PVV", "FVD", "JA21", "SGP"})
|
||||
CANONICAL_CENTRIST = frozenset({"VVD", "D66", "CDA", "NSC", "BBB", "CU"})
|
||||
|
||||
COALITION: dict[int, set[str]] = {
|
||||
2016: {"VVD", "PvdA"},
|
||||
2017: {"VVD", "PvdA"},
|
||||
2018: {"VVD", "CDA", "D66", "CU"},
|
||||
2019: {"VVD", "CDA", "D66", "CU"},
|
||||
2020: {"VVD", "CDA", "D66", "CU"},
|
||||
2021: {"VVD", "CDA", "D66", "CU"},
|
||||
2022: {"VVD", "D66", "CDA", "CU"},
|
||||
2023: {"VVD", "D66", "CDA", "CU"},
|
||||
2024: {"PVV", "VVD", "NSC", "BBB"},
|
||||
2025: {"PVV", "VVD", "NSC", "BBB"},
|
||||
2026: {"PVV", "VVD", "NSC", "BBB"},
|
||||
}
|
||||
|
||||
POLITICAL_EVENTS: list[dict[str, Any]] = [
|
||||
{"quarter": "2021-Q1", "label": "Rutte IV\nelection",
|
||||
"date": "Mar 2021", "category": "dutch"},
|
||||
@@ -70,50 +55,6 @@ logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(mess
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def build_party_name_map(con: duckdb.DuckDBPyConnection) -> dict[str, str]:
|
||||
rows = con.execute("""
|
||||
SELECT mp_name, party, van, tot_en_met
|
||||
FROM mp_metadata
|
||||
WHERE party IS NOT NULL
|
||||
ORDER BY tot_en_met DESC NULLS LAST, van DESC NULLS LAST
|
||||
""").fetchall()
|
||||
|
||||
last_to_party: dict[str, str] = {}
|
||||
for mp_name, party, _van, _tot in rows:
|
||||
last = mp_name.split(",")[0].strip()
|
||||
if last not in last_to_party:
|
||||
last_to_party[last] = party
|
||||
return last_to_party
|
||||
|
||||
|
||||
def parse_lead_submitter(
|
||||
title: str, name_party_map: dict[str, str]
|
||||
) -> tuple[str | None, str | None]:
|
||||
if not title:
|
||||
return None, None
|
||||
|
||||
patterns = [
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+het\s+lid\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+de\s+leden\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"Amendement\s+van\s+het\s+lid\s+(.+?)\s+over\b",
|
||||
r"Amendement\s+van\s+de\s+leden\s+(.+?)\s+over\b",
|
||||
]
|
||||
|
||||
for pat in patterns:
|
||||
m = re.search(pat, title)
|
||||
if m:
|
||||
submitter_str = m.group(1).strip()
|
||||
parts = submitter_str.split(" en ")
|
||||
first_name = parts[0].strip()
|
||||
first_name = re.sub(r"\s+c\.s\.", "", first_name).strip()
|
||||
if not first_name:
|
||||
continue
|
||||
party = name_party_map.get(first_name)
|
||||
return first_name, party
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def fetch_rw_motions(con: duckdb.DuckDBPyConnection) -> list[dict[str, Any]]:
|
||||
rows = con.execute("""
|
||||
SELECT
|
||||
@@ -146,11 +87,6 @@ def fetch_rw_motions(con: duckdb.DuckDBPyConnection) -> list[dict[str, Any]]:
|
||||
return result
|
||||
|
||||
|
||||
def quarter_sort_key(quarter_str: str) -> tuple[int, int]:
|
||||
parts = quarter_str.split("-Q")
|
||||
return (int(parts[0]), int(parts[1]))
|
||||
|
||||
|
||||
def aggregate_quarterly(data: list[dict]) -> dict[str, dict]:
|
||||
quarterly: dict[str, dict[str, list]] = defaultdict(
|
||||
lambda: {"all_cs": []}
|
||||
|
||||
@@ -17,18 +17,17 @@ from typing import Any
|
||||
|
||||
import duckdb
|
||||
|
||||
ROOT = Path(__file__).parent.parent.parent.resolve()
|
||||
from analysis.right_wing.common import ROOT
|
||||
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from analysis.config import CANONICAL_LEFT, CANONICAL_RIGHT
|
||||
from analysis.right_wing.common import CANONICAL_CENTRIST
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Centrist parties for cross-ideological metrics
|
||||
CANONICAL_CENTRIST = frozenset({"VVD", "D66", "CDA", "NSC", "BBB", "CU"})
|
||||
|
||||
|
||||
def _load_keywords(keywords_path: str) -> tuple[list[str], list[str]]:
|
||||
"""Load right-wing and left-wing keywords from JSON."""
|
||||
|
||||
@@ -0,0 +1,187 @@
|
||||
"""Shared constants and helpers for right-wing motion analysis.
|
||||
|
||||
Extracted from 6+ files to eliminate code duplication. All Overton analysis
|
||||
scripts should import from here instead of defining their own copies.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import re
|
||||
import math
|
||||
from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
import numpy as np
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Paths
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
DB_PATH = str(ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = ROOT / "reports" / "overton_window"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Party sets
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
CANONICAL_LEFT = frozenset({"SP", "PvdA", "GL", "GroenLinks", "GroenLinks-PvdA", "DENK", "PvdD", "Volt"})
|
||||
CANONICAL_RIGHT = frozenset({"PVV", "FVD", "JA21", "SGP"})
|
||||
CANONICAL_CENTRIST = frozenset({"VVD", "D66", "CDA", "NSC", "BBB", "CU"})
|
||||
CANONICAL_CENTRIST_STRICT = frozenset({"D66", "CDA", "NSC", "CU"})
|
||||
|
||||
CANONICAL_LEFT_SET = set(CANONICAL_LEFT)
|
||||
CANONICAL_RIGHT_SET = set(CANONICAL_RIGHT)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Time periods
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
YEAR_MIN, YEAR_MAX = 2016, 2026
|
||||
BREAK_YEAR = 2024
|
||||
SCHOOF_START_DATE = "2024-07-01"
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Coalition composition
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
RUTTE_IV_COALITION: set[str] = {"VVD", "D66", "CDA", "CU"}
|
||||
SCHOOF_COALITION: set[str] = {"PVV", "VVD", "NSC", "BBB"}
|
||||
|
||||
COALITION: dict[int, set[str]] = {
|
||||
2016: {"VVD", "PvdA"},
|
||||
2017: {"VVD", "PvdA"},
|
||||
2018: {"VVD", "CDA", "D66", "CU"},
|
||||
2019: {"VVD", "CDA", "D66", "CU"},
|
||||
2020: {"VVD", "CDA", "D66", "CU"},
|
||||
2021: {"VVD", "CDA", "D66", "CU"},
|
||||
2022: {"VVD", "D66", "CDA", "CU"},
|
||||
2023: {"VVD", "D66", "CDA", "CU"},
|
||||
2024: SCHOOF_COALITION,
|
||||
2025: SCHOOF_COALITION,
|
||||
2026: SCHOOF_COALITION,
|
||||
}
|
||||
|
||||
COALITION_NOTE = (
|
||||
"2016-2017: Rutte II (VVD/PvdA). "
|
||||
"2018-2021: Rutte III (VVD/CDA/D66/CU). "
|
||||
"2022-2023: Rutte IV (VVD/D66/CDA/CU). "
|
||||
"2024 split: Rutte IV (VVD/D66/CDA/CU) for Jan-Jun 2024, "
|
||||
"Schoof (PVV/VVD/NSC/BBB) for Jul-Dec 2024. "
|
||||
"2025-2026: Schoof (PVV/VVD/NSC/BBB). "
|
||||
"Period detection uses motion date, not just year."
|
||||
)
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Database helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _conn(db_path: str | None = None, read_only: bool = True) -> duckdb.DuckDBPyConnection:
|
||||
"""Open a DuckDB connection to the motions database."""
|
||||
return duckdb.connect(db_path or DB_PATH, read_only=read_only)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Statistical helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def cohens_d(x: np.ndarray, y: np.ndarray) -> float:
|
||||
"""Cohen's d effect size (positive when y > x)."""
|
||||
pooled = np.sqrt((np.var(x, ddof=1) + np.var(y, ddof=1)) / 2)
|
||||
if pooled == 0:
|
||||
return 0.0
|
||||
return (np.mean(y) - np.mean(x)) / pooled
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Motion metadata helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def build_party_name_map(con: duckdb.DuckDBPyConnection) -> dict[str, str]:
|
||||
"""Build mapping: last name -> party from mp_metadata."""
|
||||
rows = con.execute("""
|
||||
SELECT mp_name, party, van, tot_en_met
|
||||
FROM mp_metadata
|
||||
WHERE party IS NOT NULL
|
||||
ORDER BY tot_en_met DESC NULLS LAST, van DESC NULLS LAST
|
||||
""").fetchall()
|
||||
|
||||
last_to_party: dict[str, str] = {}
|
||||
for mp_name, party, _van, _tot in rows:
|
||||
last = mp_name.split(",")[0].strip()
|
||||
if last not in last_to_party:
|
||||
last_to_party[last] = party
|
||||
return last_to_party
|
||||
|
||||
|
||||
def parse_lead_submitter(
|
||||
title: str, name_party_map: dict[str, str]
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Parse the lead submitter from a motion title and map to party.
|
||||
|
||||
Returns (parsed_name, party) or (None, None).
|
||||
"""
|
||||
if not title:
|
||||
return None, None
|
||||
|
||||
patterns = [
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+het\s+lid\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+de\s+leden\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"Amendement\s+van\s+het\s+lid\s+(.+?)\s+over\b",
|
||||
r"Amendement\s+van\s+de\s+leden\s+(.+?)\s+over\b",
|
||||
]
|
||||
|
||||
for pat in patterns:
|
||||
m = re.search(pat, title)
|
||||
if m:
|
||||
submitter_str = m.group(1).strip()
|
||||
parts = submitter_str.split(" en ")
|
||||
first_name = parts[0].strip()
|
||||
first_name = re.sub(r"\s+c\.s\.", "", first_name).strip()
|
||||
if not first_name:
|
||||
continue
|
||||
party = name_party_map.get(first_name)
|
||||
return first_name, party
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def motion_passed(voting_results: dict | None) -> bool:
|
||||
"""Check if a motion passed based on voting_results JSON."""
|
||||
if not voting_results:
|
||||
return False
|
||||
if isinstance(voting_results, str):
|
||||
try:
|
||||
import json
|
||||
voting_results = json.loads(voting_results)
|
||||
except (ValueError, TypeError):
|
||||
return False
|
||||
return voting_results.get("result") == "aangenomen"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Temporal helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def quarter_sort_key(q: str) -> tuple[int, int]:
|
||||
"""Sort key for quarter strings like '2024-Q1'."""
|
||||
year = int(q[:4])
|
||||
quarter = int(q[-1])
|
||||
return (year, quarter)
|
||||
|
||||
|
||||
def find_inflection_point(
|
||||
quarters: list[str], values: list[float], threshold: float = 0.4
|
||||
) -> str | None:
|
||||
"""Find the first quarter where the smoothed value exceeds the threshold."""
|
||||
if len(quarters) < 3:
|
||||
return None
|
||||
for i in range(1, len(quarters) - 1):
|
||||
avg = (values[i - 1] + values[i] + values[i + 1]) / 3
|
||||
if avg > threshold:
|
||||
return quarters[i]
|
||||
return None
|
||||
@@ -13,19 +13,14 @@ from pathlib import Path
|
||||
|
||||
import duckdb
|
||||
|
||||
ROOT = Path(__file__).parent.parent.parent.resolve()
|
||||
from analysis.right_wing.common import ROOT, _conn
|
||||
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DB_PATH = ROOT / "data" / "motions.db"
|
||||
|
||||
|
||||
def _conn():
|
||||
return duckdb.connect(str(DB_PATH), read_only=True)
|
||||
|
||||
|
||||
def print_section(title: str) -> None:
|
||||
print(f"\n{'=' * 70}")
|
||||
|
||||
@@ -22,6 +22,11 @@ ROOT = Path(__file__).parent.parent.parent.resolve()
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from analysis.right_wing.common import (
|
||||
CANONICAL_CENTRIST_STRICT, BREAK_YEAR, YEAR_MIN, YEAR_MAX,
|
||||
DB_PATH, REPORTS_DIR, _conn, cohens_d,
|
||||
)
|
||||
|
||||
import duckdb
|
||||
import matplotlib
|
||||
|
||||
@@ -34,15 +39,8 @@ from analysis.config import CANONICAL_LEFT, PARTY_COLOURS, _PARTY_NORMALIZE
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DB_PATH = str(ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = ROOT / "reports" / "overton_window"
|
||||
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
BREAK_YEAR = 2024
|
||||
YEAR_MIN, YEAR_MAX = 2016, 2026
|
||||
|
||||
CANONICAL_CENTRIST_STRICT = frozenset({"D66", "CDA", "CU", "NSC"})
|
||||
|
||||
LEFT_PARTY_DISPLAY_ORDER = [
|
||||
"SP",
|
||||
"GroenLinks-PvdA",
|
||||
@@ -52,17 +50,6 @@ LEFT_PARTY_DISPLAY_ORDER = [
|
||||
]
|
||||
|
||||
|
||||
def _conn(read_only: bool = True) -> duckdb.DuckDBPyConnection:
|
||||
return duckdb.connect(DB_PATH, read_only=read_only)
|
||||
|
||||
|
||||
def cohens_d(x: np.ndarray, y: np.ndarray) -> float:
|
||||
pooled = np.sqrt((np.var(x, ddof=1) + np.var(y, ddof=1)) / 2)
|
||||
if pooled == 0:
|
||||
return 0.0
|
||||
return (np.mean(y) - np.mean(x)) / pooled
|
||||
|
||||
|
||||
def query_yearly_support() -> dict[int, dict]:
|
||||
"""Query yearly averages of left_support_mp and centrist_support_strict."""
|
||||
con = _conn()
|
||||
|
||||
@@ -9,16 +9,16 @@ from __future__ import annotations
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).parent.parent.parent.resolve()
|
||||
from analysis.right_wing.common import ROOT
|
||||
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
import duckdb
|
||||
|
||||
from analysis.config import CANONICAL_LEFT
|
||||
from analysis.right_wing.common import CANONICAL_CENTRIST, CANONICAL_CENTRIST_STRICT
|
||||
|
||||
CANONICAL_CENTRIST = frozenset({"VVD", "D66", "CDA", "NSC", "BBB", "CU"})
|
||||
CANONICAL_CENTRIST_STRICT = frozenset({"D66", "CDA", "CU", "NSC"})
|
||||
CANONICAL_CENTER_RIGHT = frozenset({"VVD", "BBB"})
|
||||
|
||||
COLUMNS = [
|
||||
|
||||
@@ -36,13 +36,16 @@ matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
from analysis.config import CANONICAL_LEFT, CANONICAL_RIGHT, PARTY_COLOURS
|
||||
|
||||
CANONICAL_CENTRIST = frozenset({"VVD", "D66", "CDA", "NSC", "BBB", "CU"})
|
||||
from analysis.right_wing.common import (
|
||||
CANONICAL_CENTRIST, COALITION, COALITION_NOTE, RUTTE_IV_COALITION,
|
||||
SCHOOF_COALITION, SCHOOF_START_DATE, BREAK_YEAR, YEAR_MIN, YEAR_MAX,
|
||||
DB_PATH, REPORTS_DIR, _conn, cohens_d, build_party_name_map,
|
||||
parse_lead_submitter,
|
||||
)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DB_PATH = str(ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = ROOT / "reports" / "overton_window"
|
||||
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
CANONICAL_CENTRIST_SET = set(CANONICAL_CENTRIST)
|
||||
@@ -62,49 +65,8 @@ def _extremity_bucket(score: float) -> str:
|
||||
CANONICAL_LEFT_SET = set(CANONICAL_LEFT)
|
||||
CANONICAL_RIGHT_SET = set(CANONICAL_RIGHT)
|
||||
|
||||
RUTTE_IV_COALITION: set[str] = {"VVD", "D66", "CDA", "CU"}
|
||||
SCHOOF_COALITION: set[str] = {"PVV", "VVD", "NSC", "BBB"}
|
||||
|
||||
COALITION: dict[int, set[str]] = {
|
||||
2016: {"VVD", "PvdA"},
|
||||
2017: {"VVD", "PvdA"},
|
||||
2018: {"VVD", "CDA", "D66", "CU"},
|
||||
2019: {"VVD", "CDA", "D66", "CU"},
|
||||
2020: {"VVD", "CDA", "D66", "CU"},
|
||||
2021: {"VVD", "CDA", "D66", "CU"},
|
||||
2022: {"VVD", "D66", "CDA", "CU"},
|
||||
2023: {"VVD", "D66", "CDA", "CU"},
|
||||
2024: SCHOOF_COALITION,
|
||||
2025: SCHOOF_COALITION,
|
||||
2026: SCHOOF_COALITION,
|
||||
}
|
||||
|
||||
SCHOOF_START_DATE = "2024-07-01"
|
||||
|
||||
COALITION_NOTE = (
|
||||
"2016-2017: Rutte II (VVD/PvdA). "
|
||||
"2018-2021: Rutte III (VVD/CDA/D66/CU). "
|
||||
"2022-2023: Rutte IV (VVD/D66/CDA/CU). "
|
||||
"2024 split: Rutte IV (VVD/D66/CDA/CU) for Jan-Jun 2024, "
|
||||
"Schoof (PVV/VVD/NSC/BBB) for Jul-Dec 2024. "
|
||||
"2025-2026: Schoof (PVV/VVD/NSC/BBB). "
|
||||
"Period detection uses motion date, not just year."
|
||||
)
|
||||
|
||||
YEAR_MIN, YEAR_MAX = 2016, 2026
|
||||
BREAK_YEAR = 2024
|
||||
|
||||
|
||||
def _conn(read_only: bool = True) -> duckdb.DuckDBPyConnection:
|
||||
return duckdb.connect(DB_PATH, read_only=read_only)
|
||||
|
||||
|
||||
def cohens_d(x: np.ndarray, y: np.ndarray) -> float:
|
||||
"""Cohen's d effect size."""
|
||||
pooled = np.sqrt((np.var(x, ddof=1) + np.var(y, ddof=1)) / 2)
|
||||
if pooled == 0:
|
||||
return 0.0
|
||||
return (np.mean(y) - np.mean(x)) / pooled
|
||||
|
||||
|
||||
def compute_yearly_rw_metrics(con: duckdb.DuckDBPyConnection) -> dict[int, dict]:
|
||||
@@ -246,55 +208,6 @@ def _support_ratio(
|
||||
return supportive / total
|
||||
|
||||
|
||||
def build_party_name_map(con: duckdb.DuckDBPyConnection) -> dict[str, str]:
|
||||
"""Build mapping: last name -> party from mp_metadata."""
|
||||
rows = con.execute("""
|
||||
SELECT mp_name, party, van, tot_en_met
|
||||
FROM mp_metadata
|
||||
WHERE party IS NOT NULL
|
||||
ORDER BY tot_en_met DESC NULLS LAST, van DESC NULLS LAST
|
||||
""").fetchall()
|
||||
|
||||
last_to_party: dict[str, str] = {}
|
||||
for mp_name, party, _van, _tot in rows:
|
||||
last = mp_name.split(",")[0].strip()
|
||||
if last not in last_to_party:
|
||||
last_to_party[last] = party
|
||||
return last_to_party
|
||||
|
||||
|
||||
def parse_lead_submitter(
|
||||
title: str, name_party_map: dict[str, str]
|
||||
) -> tuple[str | None, str | None]:
|
||||
"""Parse the lead submitter from a motion title and map to party.
|
||||
|
||||
Returns (parsed_name, party) or (None, None).
|
||||
"""
|
||||
if not title:
|
||||
return None, None
|
||||
|
||||
patterns = [
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+het\s+lid\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+de\s+leden\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"Amendement\s+van\s+het\s+lid\s+(.+?)\s+over\b",
|
||||
r"Amendement\s+van\s+de\s+leden\s+(.+?)\s+over\b",
|
||||
]
|
||||
|
||||
for pat in patterns:
|
||||
m = re.search(pat, title)
|
||||
if m:
|
||||
submitter_str = m.group(1).strip()
|
||||
parts = submitter_str.split(" en ")
|
||||
first_name = parts[0].strip()
|
||||
first_name = re.sub(r"\s+c\.s\.", "", first_name).strip()
|
||||
if not first_name:
|
||||
continue
|
||||
party = name_party_map.get(first_name)
|
||||
return first_name, party
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def compute_opposition_metrics(
|
||||
yearly_raw: dict[int, dict], name_party_map: dict[str, str]
|
||||
) -> dict[int, dict]:
|
||||
|
||||
@@ -25,7 +25,8 @@ import numpy as np
|
||||
|
||||
matplotlib.use("Agg")
|
||||
|
||||
ROOT = Path(__file__).parent.parent.parent.resolve()
|
||||
from analysis.right_wing.common import ROOT, DB_PATH, REPORTS_DIR
|
||||
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
@@ -42,9 +43,6 @@ CANONICAL_CENTRIST = frozenset(
|
||||
{"VVD", "D66", "CDA", "NSC", "BBB", "CU", "ChristenUnie"}
|
||||
)
|
||||
|
||||
DB_PATH = str(ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = ROOT / "reports" / "overton_window"
|
||||
|
||||
|
||||
def _normalize_party(raw: str) -> str:
|
||||
"""Normalize a raw party name to its canonical abbreviation."""
|
||||
|
||||
@@ -28,18 +28,18 @@ ROOT = Path(__file__).parent.parent.parent.resolve()
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
from analysis.right_wing.common import (
|
||||
BREAK_YEAR, YEAR_MIN, YEAR_MAX, DB_PATH, REPORTS_DIR,
|
||||
_conn, build_party_name_map,
|
||||
)
|
||||
from analysis.config import CANONICAL_RIGHT, PARTY_COLOURS, _PARTY_NORMALIZE
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DB_PATH = str(ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = ROOT / "reports" / "overton_window"
|
||||
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
RIGHT_PARTIES = sorted(CANONICAL_RIGHT)
|
||||
YEAR_MIN, YEAR_MAX = 2016, 2026
|
||||
BREAK_YEAR = 2024
|
||||
|
||||
TITLE_PATTERNS = [
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+het\s+lid\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
@@ -49,26 +49,6 @@ TITLE_PATTERNS = [
|
||||
]
|
||||
|
||||
|
||||
def _conn(read_only: bool = True) -> duckdb.DuckDBPyConnection:
|
||||
return duckdb.connect(DB_PATH, read_only=read_only)
|
||||
|
||||
|
||||
def build_party_name_map(con: duckdb.DuckDBPyConnection) -> dict[str, str]:
|
||||
rows = con.execute("""
|
||||
SELECT mp_name, party, van, tot_en_met
|
||||
FROM mp_metadata
|
||||
WHERE party IS NOT NULL
|
||||
ORDER BY tot_en_met DESC NULLS LAST, van DESC NULLS LAST
|
||||
""").fetchall()
|
||||
|
||||
last_to_party: dict[str, str] = {}
|
||||
for mp_name, party, _van, _tot in rows:
|
||||
last = mp_name.split(",")[0].strip()
|
||||
if last not in last_to_party:
|
||||
last_to_party[last] = party
|
||||
return last_to_party
|
||||
|
||||
|
||||
def parse_submitter_party(title: str, name_party_map: dict[str, str]) -> str | None:
|
||||
if not title:
|
||||
return None
|
||||
|
||||
@@ -45,31 +45,18 @@ PROJECT_ROOT = Path(__file__).resolve().parent.parent.parent
|
||||
if str(PROJECT_ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
|
||||
from analysis.right_wing.common import (
|
||||
BREAK_YEAR, COALITION, DB_PATH, REPORTS_DIR,
|
||||
build_party_name_map as build_name_party_map, parse_lead_submitter,
|
||||
)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DB_PATH = str(PROJECT_ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = PROJECT_ROOT / "reports" / "overton_window"
|
||||
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
RANDOM_SEED = 42
|
||||
|
||||
BREAK_YEAR = 2024
|
||||
|
||||
COALITION: dict[int, set[str]] = {
|
||||
2016: {"VVD", "PvdA"},
|
||||
2017: {"VVD", "PvdA"},
|
||||
2018: {"VVD", "CDA", "D66", "CU"},
|
||||
2019: {"VVD", "CDA", "D66", "CU"},
|
||||
2020: {"VVD", "CDA", "D66", "CU"},
|
||||
2021: {"VVD", "CDA", "D66", "CU"},
|
||||
2022: {"VVD", "D66", "CDA", "CU"},
|
||||
2023: {"VVD", "D66", "CDA", "CU"},
|
||||
2024: {"PVV", "VVD", "NSC", "BBB"},
|
||||
2025: {"PVV", "VVD", "NSC", "BBB"},
|
||||
2026: {"PVV", "VVD", "NSC", "BBB"},
|
||||
}
|
||||
|
||||
RIGHT_WING_PARTIES = {"PVV", "FVD", "JA21", "SGP"}
|
||||
|
||||
CATEGORY_SHORT = {
|
||||
@@ -89,50 +76,6 @@ CATEGORY_SHORT = {
|
||||
}
|
||||
|
||||
|
||||
def build_name_party_map(con: duckdb.DuckDBPyConnection) -> dict[str, str]:
|
||||
rows = con.execute("""
|
||||
SELECT mp_name, party, van, tot_en_met
|
||||
FROM mp_metadata
|
||||
WHERE party IS NOT NULL
|
||||
ORDER BY tot_en_met DESC NULLS LAST, van DESC NULLS LAST
|
||||
""").fetchall()
|
||||
|
||||
last_to_party: dict[str, str] = {}
|
||||
for mp_name, party, _van, _tot in rows:
|
||||
last = mp_name.split(",")[0].strip()
|
||||
if last not in last_to_party:
|
||||
last_to_party[last] = party
|
||||
return last_to_party
|
||||
|
||||
|
||||
def parse_lead_submitter(
|
||||
title: str, name_party_map: dict[str, str]
|
||||
) -> tuple[str | None, str | None]:
|
||||
if not title:
|
||||
return None, None
|
||||
|
||||
patterns = [
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+het\s+lid\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+de\s+leden\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"Amendement\s+van\s+het\s+lid\s+(.+?)\s+over\b",
|
||||
r"Amendement\s+van\s+de\s+leden\s+(.+?)\s+over\b",
|
||||
]
|
||||
|
||||
for pat in patterns:
|
||||
m = re.search(pat, title)
|
||||
if m:
|
||||
submitter_str = m.group(1).strip()
|
||||
parts = submitter_str.split(" en ")
|
||||
first_name = parts[0].strip()
|
||||
first_name = re.sub(r"\s+c\.s\.", "", first_name).strip()
|
||||
if not first_name:
|
||||
continue
|
||||
party = name_party_map.get(first_name)
|
||||
return first_name, party
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def load_model_data(
|
||||
db_path: str,
|
||||
) -> tuple[list[dict[str, Any]], int, int]:
|
||||
|
||||
@@ -29,76 +29,17 @@ import duckdb
|
||||
import numpy as np
|
||||
from scipy.stats import chi2
|
||||
|
||||
from analysis.right_wing.common import (
|
||||
BREAK_YEAR, COALITION, DB_PATH, REPORTS_DIR,
|
||||
build_party_name_map, parse_lead_submitter,
|
||||
)
|
||||
from analysis.config import CANONICAL_RIGHT
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DB_PATH = str(PROJECT_ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = PROJECT_ROOT / "reports" / "overton_window"
|
||||
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
BREAK_YEAR = 2024
|
||||
|
||||
COALITION: dict[int, set[str]] = {
|
||||
2016: {"VVD", "PvdA"},
|
||||
2017: {"VVD", "PvdA"},
|
||||
2018: {"VVD", "CDA", "D66", "CU"},
|
||||
2019: {"VVD", "CDA", "D66", "CU"},
|
||||
2020: {"VVD", "CDA", "D66", "CU"},
|
||||
2021: {"VVD", "CDA", "D66", "CU"},
|
||||
2022: {"VVD", "D66", "CDA", "CU"},
|
||||
2023: {"VVD", "D66", "CDA", "CU"},
|
||||
2024: {"PVV", "VVD", "NSC", "BBB"},
|
||||
2025: {"PVV", "VVD", "NSC", "BBB"},
|
||||
2026: {"PVV", "VVD", "NSC", "BBB"},
|
||||
}
|
||||
|
||||
|
||||
def build_party_name_map(con: duckdb.DuckDBPyConnection) -> dict[str, str]:
|
||||
rows = con.execute("""
|
||||
SELECT mp_name, party, van, tot_en_met
|
||||
FROM mp_metadata
|
||||
WHERE party IS NOT NULL
|
||||
ORDER BY tot_en_met DESC NULLS LAST, van DESC NULLS LAST
|
||||
""").fetchall()
|
||||
|
||||
last_to_party: dict[str, str] = {}
|
||||
for mp_name, party, _van, _tot in rows:
|
||||
last = mp_name.split(",")[0].strip()
|
||||
if last not in last_to_party:
|
||||
last_to_party[last] = party
|
||||
return last_to_party
|
||||
|
||||
|
||||
def parse_lead_submitter(
|
||||
title: str, name_party_map: dict[str, str]
|
||||
) -> tuple[str | None, str | None]:
|
||||
if not title:
|
||||
return None, None
|
||||
|
||||
patterns = [
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+het\s+lid\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+de\s+leden\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"Amendement\s+van\s+het\s+lid\s+(.+?)\s+over\b",
|
||||
r"Amendement\s+van\s+de\s+leden\s+(.+?)\s+over\b",
|
||||
]
|
||||
|
||||
for pat in patterns:
|
||||
m = re.search(pat, title)
|
||||
if m:
|
||||
submitter_str = m.group(1).strip()
|
||||
parts = submitter_str.split(" en ")
|
||||
first_name = parts[0].strip()
|
||||
first_name = re.sub(r"\s+c\.s\.", "", first_name).strip()
|
||||
if not first_name:
|
||||
continue
|
||||
party = name_party_map.get(first_name)
|
||||
return first_name, party
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def motion_passed(voting: dict | None, winning_margin: float | None = None) -> bool | None:
|
||||
if voting is None:
|
||||
voting = {}
|
||||
|
||||
@@ -23,7 +23,8 @@ import numpy as np
|
||||
|
||||
matplotlib.use("Agg")
|
||||
|
||||
ROOT = Path(__file__).parent.parent.parent.resolve()
|
||||
from analysis.right_wing.common import ROOT, DB_PATH, REPORTS_DIR
|
||||
|
||||
if str(ROOT) not in sys.path:
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
@@ -40,8 +41,6 @@ CANONICAL_CENTRIST = frozenset(
|
||||
{"VVD", "D66", "CDA", "NSC", "BBB", "CU", "ChristenUnie"}
|
||||
)
|
||||
|
||||
DB_PATH = str(ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = ROOT / "reports" / "overton_window"
|
||||
OUTPUT_PATH = str(REPORTS_DIR / "svd_trajectory_figure.png")
|
||||
|
||||
CENTRIST_DISPLAY = ["VVD", "D66", "CDA", "NSC", "BBB", "CU"]
|
||||
|
||||
@@ -32,75 +32,16 @@ import numpy as np
|
||||
ROOT = Path(__file__).parent.parent.parent.resolve()
|
||||
sys.path.insert(0, str(ROOT))
|
||||
|
||||
DB_PATH = str(ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = ROOT / "reports" / "overton_window"
|
||||
from analysis.right_wing.common import (
|
||||
CANONICAL_CENTRIST, COALITION, DB_PATH, REPORTS_DIR,
|
||||
build_party_name_map, parse_lead_submitter, quarter_sort_key,
|
||||
)
|
||||
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
CANONICAL_RIGHT = frozenset({"PVV", "FVD", "JA21", "SGP"})
|
||||
CANONICAL_CENTRIST = frozenset({"VVD", "D66", "CDA", "NSC", "BBB", "CU"})
|
||||
|
||||
COALITION: dict[int, set[str]] = {
|
||||
2016: {"VVD", "PvdA"},
|
||||
2017: {"VVD", "PvdA"},
|
||||
2018: {"VVD", "CDA", "D66", "CU"},
|
||||
2019: {"VVD", "CDA", "D66", "CU"},
|
||||
2020: {"VVD", "CDA", "D66", "CU"},
|
||||
2021: {"VVD", "CDA", "D66", "CU"},
|
||||
2022: {"VVD", "D66", "CDA", "CU"},
|
||||
2023: {"VVD", "D66", "CDA", "CU"},
|
||||
2024: {"PVV", "VVD", "NSC", "BBB"},
|
||||
2025: {"PVV", "VVD", "NSC", "BBB"},
|
||||
2026: {"PVV", "VVD", "NSC", "BBB"},
|
||||
}
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def build_party_name_map(con: duckdb.DuckDBPyConnection) -> dict[str, str]:
|
||||
rows = con.execute("""
|
||||
SELECT mp_name, party, van, tot_en_met
|
||||
FROM mp_metadata
|
||||
WHERE party IS NOT NULL
|
||||
ORDER BY tot_en_met DESC NULLS LAST, van DESC NULLS LAST
|
||||
""").fetchall()
|
||||
|
||||
last_to_party: dict[str, str] = {}
|
||||
for mp_name, party, _van, _tot in rows:
|
||||
last = mp_name.split(",")[0].strip()
|
||||
if last not in last_to_party:
|
||||
last_to_party[last] = party
|
||||
return last_to_party
|
||||
|
||||
|
||||
def parse_lead_submitter(
|
||||
title: str, name_party_map: dict[str, str]
|
||||
) -> tuple[str | None, str | None]:
|
||||
if not title:
|
||||
return None, None
|
||||
|
||||
patterns = [
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+het\s+lid\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"(?:Gewijzigde|Nader\s+gewijzigde)?\s*Motie\s+van\s+de\s+leden\s+(.+?)\s+(?:c\.s\.\s+)?over\b",
|
||||
r"Amendement\s+van\s+het\s+lid\s+(.+?)\s+over\b",
|
||||
r"Amendement\s+van\s+de\s+leden\s+(.+?)\s+over\b",
|
||||
]
|
||||
|
||||
for pat in patterns:
|
||||
m = re.search(pat, title)
|
||||
if m:
|
||||
submitter_str = m.group(1).strip()
|
||||
parts = submitter_str.split(" en ")
|
||||
first_name = parts[0].strip()
|
||||
first_name = re.sub(r"\s+c\.s\.", "", first_name).strip()
|
||||
if not first_name:
|
||||
continue
|
||||
party = name_party_map.get(first_name)
|
||||
return first_name, party
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
def fetch_quarterly_data(con: duckdb.DuckDBPyConnection) -> list[dict[str, Any]]:
|
||||
"""Fetch all right-wing motions with dates and metrics."""
|
||||
rows = con.execute("""
|
||||
@@ -172,12 +113,6 @@ def aggregate_quarterly(
|
||||
return dict(quarterly)
|
||||
|
||||
|
||||
def quarter_sort_key(quarter_str: str) -> tuple[int, int]:
|
||||
"""Sort key: '2019-Q3' -> (2019, 3)."""
|
||||
parts = quarter_str.split("-Q")
|
||||
return (int(parts[0]), int(parts[1]))
|
||||
|
||||
|
||||
def compute_summary(quarterly: dict) -> dict[str, dict[str, Any]]:
|
||||
"""Compute means, counts, and confidence intervals per quarter."""
|
||||
summary = {}
|
||||
|
||||
@@ -38,16 +38,13 @@ import numpy as np
|
||||
from scipy.stats import spearmanr, pearsonr, mannwhitneyu
|
||||
|
||||
from analysis.config import CANONICAL_RIGHT
|
||||
from analysis.right_wing.common import BREAK_YEAR, DB_PATH, REPORTS_DIR
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
DB_PATH = str(PROJECT_ROOT / "data" / "motions.db")
|
||||
REPORTS_DIR = PROJECT_ROOT / "reports" / "overton_window"
|
||||
REPORTS_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
BREAK_YEAR = 2024
|
||||
|
||||
QUARTILE_LABELS = [
|
||||
"Q1 [0.00\u20130.25]",
|
||||
"Q2 (0.25\u20130.50]",
|
||||
@@ -540,22 +537,19 @@ def generate_report(
|
||||
"",
|
||||
]
|
||||
|
||||
if u_p < 0.05 if isinstance(u_p := corr.get("spearman_p", 1.0), float) else False:
|
||||
pass
|
||||
else:
|
||||
if not np.isnan(post_mean) and not np.isnan(pre_mean):
|
||||
_, period_p = mannwhitneyu(pre_margins, post_margins, alternative="two-sided")
|
||||
if period_p < 0.05:
|
||||
direction = "rose" if post_mean > pre_mean else "fell"
|
||||
report.append(
|
||||
f"Voting margin {direction} significantly post-2024 "
|
||||
f"(Mann-Whitney p = {period_p:.1e}, d = {cohens_d:+.3f})."
|
||||
)
|
||||
else:
|
||||
report.append(
|
||||
f"Voting margin did not change significantly between periods "
|
||||
f"(Mann-Whitney p = {period_p:.3f})."
|
||||
)
|
||||
if not np.isnan(post_mean) and not np.isnan(pre_mean):
|
||||
_, period_p = mannwhitneyu(pre_margins, post_margins, alternative="two-sided")
|
||||
if period_p < 0.05:
|
||||
direction = "rose" if post_mean > pre_mean else "fell"
|
||||
report.append(
|
||||
f"Voting margin {direction} significantly post-2024 "
|
||||
f"(Mann-Whitney p = {period_p:.1e}, d = {cohens_d:+.3f})."
|
||||
)
|
||||
else:
|
||||
report.append(
|
||||
f"Voting margin did not change significantly between periods "
|
||||
f"(Mann-Whitney p = {period_p:.3f})."
|
||||
)
|
||||
|
||||
report += [
|
||||
"",
|
||||
|
||||
Reference in New Issue
Block a user