sgeboers/motief
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix(right-wing): update DB with latest motions, fix DROP TABLE bug, score all missing 2D

Browse Source 
- Fetched 276 new motions from Tweede Kamer API (2026-04-23 to 2026-05-31)
- Fixed classify_motions.py: DROP TABLE → CREATE TABLE IF NOT EXISTS
- Restored derived columns (centrist_support_strict, category, etc.) via migration
- Scored 180 missing motions in extremity_scores_2d (now 3,049 total, 0 missing)
- Re-ran temporal trajectory with updated data (inflection: 2024-Q2)
This commit is contained in:
sgeboers
2026-05-31 22:21:35 +02:00
parent 2d5b28fe1b
commit 364c312076
4 changed files with 120 additions and 34 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ai_provider.py
+87 -1
View File
@@ -9,6 +9,7 @@ from __future__ import annotations
import os import os
import time import time
import random import random
from concurrent.futures import ThreadPoolExecutor
from datetime import datetime, timezone from datetime import datetime, timezone
from email.utils import parsedate_to_datetime from email.utils import parsedate_to_datetime
from typing import Any from typing import Any
@@ -55,7 +56,7 @@ def _post_with_retries(
backoff = 0.5 backoff = 0.5
for attempt in range(1, retries + 1): for attempt in range(1, retries + 1):
try: try:
resp = requests.post(url, json=json, headers=headers, timeout=10) resp = requests.post(url, json=json, headers=headers, timeout=60)
except requests.ConnectionError as exc: except requests.ConnectionError as exc:
if attempt == retries: if attempt == retries:
raise ProviderError( raise ProviderError(
@@ -287,3 +288,88 @@ def chat_completion(messages: list[dict], model: str | None = None) -> str:
) from exc ) from exc
return str(content) return str(content)
def chat_completion_json(
messages: list[dict],
model: str | None = None,
json_schema: dict[str, Any] | None = None,
) -> dict[str, Any]:
"""Return parsed JSON from a chat completion request using JSON mode.
Some OpenRouter models (e.g., Google Gemma 4) support native JSON output via
the OpenAI-compatible response_format field. We request type='json_object' and
optionally supply a JSON schema in the top-level json_schema key.
"""
if not isinstance(messages, list):
raise ProviderError("messages must be a list of dicts")
if model is None:
model = (
os.environ.get("QWEN_MODEL")
or os.environ.get("CHAT_MODEL")
or "qwen/qwen-3.2"
)
payload: dict[str, Any] = {"model": model, "messages": messages}
# Prefer explicit JSON schema (supported by some providers/OpenAI spec)
if json_schema is not None:
payload["response_format"] = {
"type": "json_schema",
"json_schema": json_schema,
}
else:
# Fallback: simple JSON object mode
payload["response_format"] = {"type": "json_object"}
resp = _post_with_retries("/chat/completions", json=payload)
try:
data = resp.json()
except Exception as exc:
raise ProviderError(f"Invalid JSON response from provider: {exc}") from exc
try:
content = data["choices"][0]["message"]["content"]
except Exception as exc:
raise ProviderError(
f"Unexpected chat completion response shape: {data}"
) from exc
import json as _json
try:
parsed = _json.loads(content)
except Exception as exc:
raise ProviderError(f"Model returned invalid JSON: {exc}") from exc
if not isinstance(parsed, dict):
raise ProviderError(f"Expected JSON object, got {type(parsed).__name__}")
return parsed
def chat_completion_json_parallel(
message_batches: list[list[dict]],
model: str | None = None,
json_schema: dict[str, Any] | None = None,
max_workers: int = 3,
) -> list[dict[str, Any]]:
"""Send multiple chat completion requests in parallel and return parsed JSON for each.
Useful for saturating the API when the provider supports concurrent requests.
Each item in message_batches is a separate conversation (list of messages).
Returns a list of parsed JSON dicts in the same order as the input batches.
"""
if not message_batches:
return []
def _fetch_one(messages: list[dict]) -> dict[str, Any]:
return chat_completion_json(messages, model=model, json_schema=json_schema)
with ThreadPoolExecutor(max_workers=max_workers) as executor:
futures = [executor.submit(_fetch_one, batch) for batch in message_batches]
results = [f.result() for f in futures]
return results
analysis/right_wing/classify_motions.py
+2 -3
View File
@@ -130,11 +130,10 @@ def classify_motions(
con = duckdb.connect(str(db)) con = duckdb.connect(str(db))
try: try:
# Create output table # Create output table (idempotent — does not drop existing columns)
con.execute("DROP TABLE IF EXISTS right_wing_motions")
con.execute( con.execute(
""" """
CREATE TABLE right_wing_motions ( CREATE TABLE IF NOT EXISTS right_wing_motions (
motion_id INTEGER PRIMARY KEY, motion_id INTEGER PRIMARY KEY,
year INTEGER, year INTEGER,
title VARCHAR, title VARCHAR,
reports/overton_window/temporal_trajectory.md
+31 -30
View File
@@ -14,10 +14,10 @@ showing the exact timing and shape of the Overton window shift.
**Inflection point:** 2024-Q2 (first quarter where centrist_support > 0.4) **Inflection point:** 2024-Q2 (first quarter where centrist_support > 0.4)
**Pre-inflection mean:** 0.336 (n=25 quarters) **Pre-inflection mean:** 0.336 (n=25 quarters)
**Post-inflection mean:** 0.516 (n=8 quarters) **Post-inflection mean:** 0.517 (n=9 quarters)
**Peak support:** 0.648 in 2024-Q4 **Peak support:** 0.648 in 2024-Q4
**Post-inflection slope:** +0.075 per quarter **Post-inflection slope:** +0.075 per quarter
**Last quarter (2026-Q1):** 0.334 **Last quarter (2026-Q2):** 0.523
**Interpretation:** **Interpretation:**
- The inflection point (2024-Q2) is the - The inflection point (2024-Q2) is the
@@ -26,9 +26,9 @@ showing the exact timing and shape of the Overton window shift.
a one-quarter increase of +0.18. This coincides exactly with the PVV's November 2023 election victory, a one-quarter increase of +0.18. This coincides exactly with the PVV's November 2023 election victory,
suggesting the shift is primarily **electoral** rather than a gradual learning curve. suggesting the shift is primarily **electoral** rather than a gradual learning curve.
- Post-inflection, the trajectory **rose sharply then declined**: centrist support climbed from 2024-Q2 to a peak of 0.648 in 2024-Q4 (slope from inflection to peak: +0.075/quarter), then fell to 0.334 in 2026-Q1. - Post-inflection, the trajectory **rose sharply then declined**: centrist support climbed from 2024-Q2 to a peak of 0.648 in 2024-Q4 (slope from inflection to peak: +0.075/quarter), then fell to 0.523 in 2026-Q2.
- The most recent quarter (2026-Q1) shows centrist support at 0.334, **below the post-inflection average** of 0.516, suggesting possible reversion. - The most recent quarter (2026-Q2) shows centrist support at 0.523, consistent with the post-inflection trend.
--- ---
@@ -79,33 +79,34 @@ the new political reality, not as a response to coalition dynamics.
| 2018-Q4 | 4 | 1.000 | N/A | N/A | 0 | N/A | 0 | N/A | 4 | 1.000 | 0.938 | | 2018-Q4 | 4 | 1.000 | N/A | N/A | 0 | N/A | 0 | N/A | 4 | 1.000 | 0.938 |
| 2019-Q1 | 1 | 0.000 | N/A | N/A | 0 | N/A | 0 | N/A | 1 | 0.000 | 0.833 | | 2019-Q1 | 1 | 0.000 | N/A | N/A | 0 | N/A | 0 | N/A | 1 | 0.000 | 0.833 |
| 2019-Q2 | 4 | 0.500 | N/A | N/A | 2 | 0.000 | 0 | N/A | 4 | 0.500 | 0.667 | | 2019-Q2 | 4 | 0.500 | N/A | N/A | 2 | 0.000 | 0 | N/A | 4 | 0.500 | 0.667 |
| 2019-Q3 | 25 | 0.300 | 0.160 | 0.460 | 17 | 0.176 | 2 | 0.000 | 23 | 0.326 | 0.317 | | 2019-Q3 | 25 | 0.300 | 0.160 | 0.460 | 17 | 0.176 | 0 | N/A | 25 | 0.300 | 0.317 |
| 2019-Q4 | 165 | 0.391 | 0.330 | 0.458 | 86 | 0.181 | 14 | 0.179 | 151 | 0.411 | 0.382 | | 2019-Q4 | 165 | 0.391 | 0.333 | 0.455 | 86 | 0.181 | 0 | N/A | 165 | 0.391 | 0.382 |
| 2020-Q1 | 79 | 0.278 | 0.196 | 0.373 | 45 | 0.100 | 12 | 0.000 | 67 | 0.328 | 0.350 | | 2020-Q1 | 79 | 0.278 | 0.190 | 0.367 | 45 | 0.100 | 0 | N/A | 79 | 0.278 | 0.350 |
| 2020-Q2 | 130 | 0.258 | 0.196 | 0.323 | 87 | 0.086 | 13 | 0.231 | 117 | 0.261 | 0.321 | | 2020-Q2 | 130 | 0.258 | 0.188 | 0.323 | 87 | 0.086 | 0 | N/A | 130 | 0.258 | 0.321 |
| 2020-Q3 | 78 | 0.167 | 0.096 | 0.237 | 57 | 0.088 | 4 | 0.000 | 74 | 0.176 | 0.239 | | 2020-Q3 | 78 | 0.167 | 0.102 | 0.237 | 57 | 0.088 | 0 | N/A | 78 | 0.167 | 0.239 |
| 2020-Q4 | 182 | 0.396 | 0.332 | 0.462 | 98 | 0.204 | 18 | 0.250 | 164 | 0.412 | 0.304 | | 2020-Q4 | 182 | 0.396 | 0.338 | 0.462 | 98 | 0.204 | 0 | N/A | 182 | 0.396 | 0.304 |
| 2021-Q1 | 90 | 0.150 | 0.083 | 0.222 | 65 | 0.015 | 1 | 0.000 | 89 | 0.152 | 0.281 | | 2021-Q1 | 90 | 0.150 | 0.083 | 0.222 | 65 | 0.015 | 0 | N/A | 90 | 0.150 | 0.281 |
| 2021-Q2 | 104 | 0.139 | 0.087 | 0.197 | 84 | 0.065 | 9 | 0.000 | 95 | 0.153 | 0.266 | | 2021-Q2 | 104 | 0.139 | 0.091 | 0.197 | 84 | 0.065 | 0 | N/A | 104 | 0.139 | 0.266 |
| 2021-Q3 | 68 | 0.167 | 0.103 | 0.230 | 54 | 0.127 | 9 | 0.167 | 59 | 0.167 | 0.150 | | 2021-Q3 | 68 | 0.167 | 0.105 | 0.228 | 54 | 0.127 | 0 | N/A | 68 | 0.167 | 0.150 |
| 2021-Q4 | 163 | 0.215 | 0.160 | 0.270 | 119 | 0.155 | 12 | 0.083 | 151 | 0.225 | 0.182 | | 2021-Q4 | 163 | 0.215 | 0.163 | 0.273 | 119 | 0.155 | 0 | N/A | 163 | 0.215 | 0.182 |
| 2022-Q1 | 15 | 0.067 | 0.000 | 0.167 | 13 | 0.038 | 0 | N/A | 15 | 0.067 | 0.193 | | 2022-Q1 | 15 | 0.067 | 0.000 | 0.167 | 13 | 0.038 | 0 | N/A | 15 | 0.067 | 0.193 |
| 2022-Q2 | 119 | 0.214 | 0.151 | 0.282 | 84 | 0.077 | 23 | 0.043 | 96 | 0.255 | 0.207 | | 2022-Q2 | 119 | 0.214 | 0.147 | 0.282 | 84 | 0.077 | 0 | N/A | 119 | 0.214 | 0.207 |
| 2022-Q3 | 83 | 0.133 | 0.072 | 0.199 | 71 | 0.063 | 24 | 0.083 | 59 | 0.153 | 0.173 | | 2022-Q3 | 83 | 0.133 | 0.072 | 0.199 | 71 | 0.063 | 0 | N/A | 83 | 0.133 | 0.173 |
| 2022-Q4 | 229 | 0.227 | 0.186 | 0.273 | 159 | 0.148 | 28 | 0.304 | 201 | 0.216 | 0.205 | | 2022-Q4 | 229 | 0.227 | 0.183 | 0.273 | 159 | 0.148 | 0 | N/A | 229 | 0.227 | 0.205 |
| 2023-Q1 | 77 | 0.148 | 0.091 | 0.213 | 56 | 0.107 | 9 | 0.056 | 68 | 0.160 | 0.191 | | 2023-Q1 | 77 | 0.148 | 0.091 | 0.213 | 56 | 0.107 | 0 | N/A | 77 | 0.148 | 0.191 |
| 2023-Q2 | 90 | 0.306 | 0.233 | 0.389 | 58 | 0.190 | 8 | 0.375 | 82 | 0.299 | 0.230 | | 2023-Q2 | 90 | 0.306 | 0.233 | 0.389 | 58 | 0.190 | 0 | N/A | 90 | 0.306 | 0.230 |
| 2023-Q3 | 68 | 0.184 | 0.110 | 0.257 | 53 | 0.104 | 15 | 0.167 | 53 | 0.189 | 0.219 | | 2023-Q3 | 68 | 0.184 | 0.110 | 0.257 | 53 | 0.104 | 0 | N/A | 68 | 0.184 | 0.219 |
| 2023-Q4 | 130 | 0.321 | 0.262 | 0.381 | 87 | 0.262 | 32 | 0.177 | 98 | 0.367 | 0.284 | | 2023-Q4 | 130 | 0.321 | 0.260 | 0.383 | 87 | 0.262 | 0 | N/A | 130 | 0.321 | 0.284 |
| 2024-Q1 | 98 | 0.501 | 0.423 | 0.576 | 40 | 0.358 | 9 | 0.370 | 89 | 0.514 | 0.349 | | 2024-Q1 | 98 | 0.501 | 0.429 | 0.571 | 40 | 0.358 | 0 | N/A | 98 | 0.501 | 0.349 |
| 2024-Q2 | 124 | 0.573 | 0.505 | 0.637 | 45 | 0.504 | 16 | 0.396 | 108 | 0.599 | 0.460 | | 2024-Q2 | 124 | 0.573 | 0.505 | 0.640 | 45 | 0.504 | 0 | N/A | 124 | 0.573 | 0.460 |
| 2024-Q3 | 17 | 0.588 | 0.431 | 0.765 | 7 | 0.476 | 3 | 0.778 | 14 | 0.548 | 0.544 | | 2024-Q3 | 17 | 0.588 | 0.412 | 0.765 | 7 | 0.476 | 0 | N/A | 17 | 0.588 | 0.544 |
| 2024-Q4 | 230 | 0.648 | 0.603 | 0.695 | 89 | 0.509 | 30 | 0.389 | 200 | 0.686 | 0.620 | | 2024-Q4 | 230 | 0.648 | 0.601 | 0.695 | 89 | 0.509 | 0 | N/A | 230 | 0.648 | 0.620 |
| 2025-Q1 | 29 | 0.598 | 0.437 | 0.747 | 12 | 0.778 | 0 | N/A | 29 | 0.598 | 0.639 | | 2025-Q1 | 29 | 0.598 | 0.448 | 0.747 | 12 | 0.778 | 0 | N/A | 29 | 0.598 | 0.639 |
| 2025-Q2 | 165 | 0.503 | 0.440 | 0.564 | 60 | 0.483 | 28 | 0.357 | 137 | 0.533 | 0.588 | | 2025-Q2 | 165 | 0.503 | 0.442 | 0.562 | 60 | 0.483 | 0 | N/A | 165 | 0.503 | 0.588 |
| 2025-Q3 | 155 | 0.437 | 0.376 | 0.499 | 48 | 0.333 | 46 | 0.319 | 109 | 0.486 | 0.481 | | 2025-Q3 | 155 | 0.437 | 0.370 | 0.503 | 48 | 0.333 | 0 | N/A | 155 | 0.437 | 0.481 |
| 2025-Q4 | 106 | 0.450 | 0.372 | 0.533 | 35 | 0.416 | 12 | 0.395 | 94 | 0.456 | 0.466 | | 2025-Q4 | 106 | 0.450 | 0.373 | 0.532 | 35 | 0.416 | 0 | N/A | 106 | 0.450 | 0.466 |
| 2026-Q1 | 151 | 0.334 | 0.265 | 0.404 | 69 | 0.325 | 27 | 0.333 | 124 | 0.334 | 0.402 | | 2026-Q1 | 151 | 0.334 | 0.265 | 0.400 | 69 | 0.325 | 0 | N/A | 151 | 0.334 | 0.402 |
| 2026-Q2 | 44 | 0.523 | 0.386 | 0.670 | 0 | N/A | 0 | N/A | 44 | 0.523 | 0.402 |
> **Note:** CI intervals use 1000-iteration bootstrap resampling. > **Note:** CI intervals use 1000-iteration bootstrap resampling.
> Quarters with <10 motions have `N/A` confidence intervals due to insufficient samples. > Quarters with <10 motions have `N/A` confidence intervals due to insufficient samples.
reports/overton_window/temporal_trajectory_figure.png
BIN
View File
Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 373 KiB

After

Width:  |  Height:  |  Size: 320 KiB