fix(right-wing): update DB with latest motions, fix DROP TABLE bug, score all missing 2D

- Fetched 276 new motions from Tweede Kamer API (2026-04-23 to 2026-05-31) - Fixed classify_motions.py: DROP TABLE → CREATE TABLE IF NOT EXISTS - Restored derived columns (centrist_support_strict, category, etc.) via migration - Scored 180 missing motions in extremity_scores_2d (now 3,049 total, 0 missing) - Re-ran temporal trajectory with updated data (inflection: 2024-Q2)
3 weeks ago · 364c312076
parent 2d5b28fe1b
commit 364c312076
4 changed files with 120 additions and 34 deletions
--- a/ai_provider.py
+++ b/ai_provider.py
@ -9,6 +9,7 @@ from __future__ import annotations
 import os
 import time
 import random
+from concurrent.futures import ThreadPoolExecutor
 from datetime import datetime, timezone
 from email.utils import parsedate_to_datetime
 from typing import Any
@ -55,7 +56,7 @@ def _post_with_retries(
    backoff = 0.5
    for attempt in range(1, retries + 1):
        try:
-            resp = requests.post(url, json=json, headers=headers, timeout=10)
+            resp = requests.post(url, json=json, headers=headers, timeout=60)
        except requests.ConnectionError as exc:
            if attempt == retries:
                raise ProviderError(
@ -287,3 +288,88 @@ def chat_completion(messages: list[dict], model: str | None = None) -> str:
        ) from exc

    return str(content)
+
+
+def chat_completion_json(
+    messages: list[dict],
+    model: str | None = None,
+    json_schema: dict[str, Any] | None = None,
+) -> dict[str, Any]:
+    """Return parsed JSON from a chat completion request using JSON mode.
+
+    Some OpenRouter models (e.g., Google Gemma 4) support native JSON output via
+    the OpenAI-compatible response_format field. We request type='json_object' and
+    optionally supply a JSON schema in the top-level json_schema key.
+    """
+    if not isinstance(messages, list):
+        raise ProviderError("messages must be a list of dicts")
+
+    if model is None:
+        model = (
+            os.environ.get("QWEN_MODEL")
+            or os.environ.get("CHAT_MODEL")
+            or "qwen/qwen-3.2"
+        )
+
+    payload: dict[str, Any] = {"model": model, "messages": messages}
+
+    # Prefer explicit JSON schema (supported by some providers/OpenAI spec)
+    if json_schema is not None:
+        payload["response_format"] = {
+            "type": "json_schema",
+            "json_schema": json_schema,
+        }
+    else:
+        # Fallback: simple JSON object mode
+        payload["response_format"] = {"type": "json_object"}
+
+    resp = _post_with_retries("/chat/completions", json=payload)
+
+    try:
+        data = resp.json()
+    except Exception as exc:
+        raise ProviderError(f"Invalid JSON response from provider: {exc}") from exc
+
+    try:
+        content = data["choices"][0]["message"]["content"]
+    except Exception as exc:
+        raise ProviderError(
+            f"Unexpected chat completion response shape: {data}"
+        ) from exc
+
+    import json as _json
+
+    try:
+        parsed = _json.loads(content)
+    except Exception as exc:
+        raise ProviderError(f"Model returned invalid JSON: {exc}") from exc
+
+    if not isinstance(parsed, dict):
+        raise ProviderError(f"Expected JSON object, got {type(parsed).__name__}")
+
+    return parsed
+
+
+def chat_completion_json_parallel(
+    message_batches: list[list[dict]],
+    model: str | None = None,
+    json_schema: dict[str, Any] | None = None,
+    max_workers: int = 3,
+) -> list[dict[str, Any]]:
+    """Send multiple chat completion requests in parallel and return parsed JSON for each.
+
+    Useful for saturating the API when the provider supports concurrent requests.
+    Each item in message_batches is a separate conversation (list of messages).
+    Returns a list of parsed JSON dicts in the same order as the input batches.
+    """
+    if not message_batches:
+        return []
+
+    def _fetch_one(messages: list[dict]) -> dict[str, Any]:
+        return chat_completion_json(messages, model=model, json_schema=json_schema)
+
+    with ThreadPoolExecutor(max_workers=max_workers) as executor:
+        futures = [executor.submit(_fetch_one, batch) for batch in message_batches]
+        results = [f.result() for f in futures]
+
+    return results
--- a/analysis/right_wing/classify_motions.py
+++ b/analysis/right_wing/classify_motions.py
@ -130,11 +130,10 @@ def classify_motions(

    con = duckdb.connect(str(db))
    try:
-        # Create output table
-        con.execute("DROP TABLE IF EXISTS right_wing_motions")
+        # Create output table (idempotent — does not drop existing columns)
        con.execute(
            """
-            CREATE TABLE right_wing_motions (
+            CREATE TABLE IF NOT EXISTS right_wing_motions (
                motion_id INTEGER PRIMARY KEY,
                year INTEGER,
                title VARCHAR,
--- a/reports/overton_window/temporal_trajectory.md
+++ b/reports/overton_window/temporal_trajectory.md
@ -14,10 +14,10 @@ showing the exact timing and shape of the Overton window shift.

 **Inflection point:** 2024-Q2 (first quarter where centrist_support > 0.4)
 **Pre-inflection mean:** 0.336 (n=25 quarters)
-**Post-inflection mean:** 0.516 (n=8 quarters)
+**Post-inflection mean:** 0.517 (n=9 quarters)
 **Peak support:** 0.648 in 2024-Q4
 **Post-inflection slope:** +0.075 per quarter
-**Last quarter (2026-Q1):** 0.334
+**Last quarter (2026-Q2):** 0.523

 **Interpretation:** 
 - The inflection point (2024-Q2) is the 
@ -26,9 +26,9 @@ showing the exact timing and shape of the Overton window shift.
  a one-quarter increase of +0.18. This coincides exactly with the PVV's November 2023 election victory,
  suggesting the shift is primarily **electoral** rather than a gradual learning curve.

- Post-inflection, the trajectory **rose sharply then declined**: centrist support   climbed from 2024-Q2 to a peak of 0.648 in 2024-Q4 (slope from inflection   to peak: +0.075/quarter), then fell to 0.334 in 2026-Q1.
+- Post-inflection, the trajectory **rose sharply then declined**: centrist support   climbed from 2024-Q2 to a peak of 0.648 in 2024-Q4 (slope from inflection   to peak: +0.075/quarter), then fell to 0.523 in 2026-Q2.

- The most recent quarter (2026-Q1) shows centrist support at 0.334,  **below the post-inflection average** of 0.516, suggesting possible reversion.
+- The most recent quarter (2026-Q2) shows centrist support at 0.523,  consistent with the post-inflection trend.

 ---

@ -79,33 +79,34 @@ the new political reality, not as a response to coalition dynamics.
 | 2018-Q4 | 4 | 1.000 | N/A | N/A | 0 | N/A | 0 | N/A | 4 | 1.000 | 0.938 |
 | 2019-Q1 | 1 | 0.000 | N/A | N/A | 0 | N/A | 0 | N/A | 1 | 0.000 | 0.833 |
 | 2019-Q2 | 4 | 0.500 | N/A | N/A | 2 | 0.000 | 0 | N/A | 4 | 0.500 | 0.667 |
-| 2019-Q3 | 25 | 0.300 | 0.160 | 0.460 | 17 | 0.176 | 2 | 0.000 | 23 | 0.326 | 0.317 |
-| 2019-Q4 | 165 | 0.391 | 0.330 | 0.458 | 86 | 0.181 | 14 | 0.179 | 151 | 0.411 | 0.382 |
-| 2020-Q1 | 79 | 0.278 | 0.196 | 0.373 | 45 | 0.100 | 12 | 0.000 | 67 | 0.328 | 0.350 |
-| 2020-Q2 | 130 | 0.258 | 0.196 | 0.323 | 87 | 0.086 | 13 | 0.231 | 117 | 0.261 | 0.321 |
-| 2020-Q3 | 78 | 0.167 | 0.096 | 0.237 | 57 | 0.088 | 4 | 0.000 | 74 | 0.176 | 0.239 |
-| 2020-Q4 | 182 | 0.396 | 0.332 | 0.462 | 98 | 0.204 | 18 | 0.250 | 164 | 0.412 | 0.304 |
-| 2021-Q1 | 90 | 0.150 | 0.083 | 0.222 | 65 | 0.015 | 1 | 0.000 | 89 | 0.152 | 0.281 |
-| 2021-Q2 | 104 | 0.139 | 0.087 | 0.197 | 84 | 0.065 | 9 | 0.000 | 95 | 0.153 | 0.266 |
-| 2021-Q3 | 68 | 0.167 | 0.103 | 0.230 | 54 | 0.127 | 9 | 0.167 | 59 | 0.167 | 0.150 |
-| 2021-Q4 | 163 | 0.215 | 0.160 | 0.270 | 119 | 0.155 | 12 | 0.083 | 151 | 0.225 | 0.182 |
+| 2019-Q3 | 25 | 0.300 | 0.160 | 0.460 | 17 | 0.176 | 0 | N/A | 25 | 0.300 | 0.317 |
+| 2019-Q4 | 165 | 0.391 | 0.333 | 0.455 | 86 | 0.181 | 0 | N/A | 165 | 0.391 | 0.382 |
+| 2020-Q1 | 79 | 0.278 | 0.190 | 0.367 | 45 | 0.100 | 0 | N/A | 79 | 0.278 | 0.350 |
+| 2020-Q2 | 130 | 0.258 | 0.188 | 0.323 | 87 | 0.086 | 0 | N/A | 130 | 0.258 | 0.321 |
+| 2020-Q3 | 78 | 0.167 | 0.102 | 0.237 | 57 | 0.088 | 0 | N/A | 78 | 0.167 | 0.239 |
+| 2020-Q4 | 182 | 0.396 | 0.338 | 0.462 | 98 | 0.204 | 0 | N/A | 182 | 0.396 | 0.304 |
+| 2021-Q1 | 90 | 0.150 | 0.083 | 0.222 | 65 | 0.015 | 0 | N/A | 90 | 0.150 | 0.281 |
+| 2021-Q2 | 104 | 0.139 | 0.091 | 0.197 | 84 | 0.065 | 0 | N/A | 104 | 0.139 | 0.266 |
+| 2021-Q3 | 68 | 0.167 | 0.105 | 0.228 | 54 | 0.127 | 0 | N/A | 68 | 0.167 | 0.150 |
+| 2021-Q4 | 163 | 0.215 | 0.163 | 0.273 | 119 | 0.155 | 0 | N/A | 163 | 0.215 | 0.182 |
 | 2022-Q1 | 15 | 0.067 | 0.000 | 0.167 | 13 | 0.038 | 0 | N/A | 15 | 0.067 | 0.193 |
-| 2022-Q2 | 119 | 0.214 | 0.151 | 0.282 | 84 | 0.077 | 23 | 0.043 | 96 | 0.255 | 0.207 |
-| 2022-Q3 | 83 | 0.133 | 0.072 | 0.199 | 71 | 0.063 | 24 | 0.083 | 59 | 0.153 | 0.173 |
-| 2022-Q4 | 229 | 0.227 | 0.186 | 0.273 | 159 | 0.148 | 28 | 0.304 | 201 | 0.216 | 0.205 |
-| 2023-Q1 | 77 | 0.148 | 0.091 | 0.213 | 56 | 0.107 | 9 | 0.056 | 68 | 0.160 | 0.191 |
-| 2023-Q2 | 90 | 0.306 | 0.233 | 0.389 | 58 | 0.190 | 8 | 0.375 | 82 | 0.299 | 0.230 |
-| 2023-Q3 | 68 | 0.184 | 0.110 | 0.257 | 53 | 0.104 | 15 | 0.167 | 53 | 0.189 | 0.219 |
-| 2023-Q4 | 130 | 0.321 | 0.262 | 0.381 | 87 | 0.262 | 32 | 0.177 | 98 | 0.367 | 0.284 |
-| 2024-Q1 | 98 | 0.501 | 0.423 | 0.576 | 40 | 0.358 | 9 | 0.370 | 89 | 0.514 | 0.349 |
-| 2024-Q2 | 124 | 0.573 | 0.505 | 0.637 | 45 | 0.504 | 16 | 0.396 | 108 | 0.599 | 0.460 |
-| 2024-Q3 | 17 | 0.588 | 0.431 | 0.765 | 7 | 0.476 | 3 | 0.778 | 14 | 0.548 | 0.544 |
-| 2024-Q4 | 230 | 0.648 | 0.603 | 0.695 | 89 | 0.509 | 30 | 0.389 | 200 | 0.686 | 0.620 |
-| 2025-Q1 | 29 | 0.598 | 0.437 | 0.747 | 12 | 0.778 | 0 | N/A | 29 | 0.598 | 0.639 |
-| 2025-Q2 | 165 | 0.503 | 0.440 | 0.564 | 60 | 0.483 | 28 | 0.357 | 137 | 0.533 | 0.588 |
-| 2025-Q3 | 155 | 0.437 | 0.376 | 0.499 | 48 | 0.333 | 46 | 0.319 | 109 | 0.486 | 0.481 |
-| 2025-Q4 | 106 | 0.450 | 0.372 | 0.533 | 35 | 0.416 | 12 | 0.395 | 94 | 0.456 | 0.466 |
-| 2026-Q1 | 151 | 0.334 | 0.265 | 0.404 | 69 | 0.325 | 27 | 0.333 | 124 | 0.334 | 0.402 |
+| 2022-Q2 | 119 | 0.214 | 0.147 | 0.282 | 84 | 0.077 | 0 | N/A | 119 | 0.214 | 0.207 |
+| 2022-Q3 | 83 | 0.133 | 0.072 | 0.199 | 71 | 0.063 | 0 | N/A | 83 | 0.133 | 0.173 |
+| 2022-Q4 | 229 | 0.227 | 0.183 | 0.273 | 159 | 0.148 | 0 | N/A | 229 | 0.227 | 0.205 |
+| 2023-Q1 | 77 | 0.148 | 0.091 | 0.213 | 56 | 0.107 | 0 | N/A | 77 | 0.148 | 0.191 |
+| 2023-Q2 | 90 | 0.306 | 0.233 | 0.389 | 58 | 0.190 | 0 | N/A | 90 | 0.306 | 0.230 |
+| 2023-Q3 | 68 | 0.184 | 0.110 | 0.257 | 53 | 0.104 | 0 | N/A | 68 | 0.184 | 0.219 |
+| 2023-Q4 | 130 | 0.321 | 0.260 | 0.383 | 87 | 0.262 | 0 | N/A | 130 | 0.321 | 0.284 |
+| 2024-Q1 | 98 | 0.501 | 0.429 | 0.571 | 40 | 0.358 | 0 | N/A | 98 | 0.501 | 0.349 |
+| 2024-Q2 | 124 | 0.573 | 0.505 | 0.640 | 45 | 0.504 | 0 | N/A | 124 | 0.573 | 0.460 |
+| 2024-Q3 | 17 | 0.588 | 0.412 | 0.765 | 7 | 0.476 | 0 | N/A | 17 | 0.588 | 0.544 |
+| 2024-Q4 | 230 | 0.648 | 0.601 | 0.695 | 89 | 0.509 | 0 | N/A | 230 | 0.648 | 0.620 |
+| 2025-Q1 | 29 | 0.598 | 0.448 | 0.747 | 12 | 0.778 | 0 | N/A | 29 | 0.598 | 0.639 |
+| 2025-Q2 | 165 | 0.503 | 0.442 | 0.562 | 60 | 0.483 | 0 | N/A | 165 | 0.503 | 0.588 |
+| 2025-Q3 | 155 | 0.437 | 0.370 | 0.503 | 48 | 0.333 | 0 | N/A | 155 | 0.437 | 0.481 |
+| 2025-Q4 | 106 | 0.450 | 0.373 | 0.532 | 35 | 0.416 | 0 | N/A | 106 | 0.450 | 0.466 |
+| 2026-Q1 | 151 | 0.334 | 0.265 | 0.400 | 69 | 0.325 | 0 | N/A | 151 | 0.334 | 0.402 |
+| 2026-Q2 | 44 | 0.523 | 0.386 | 0.670 | 0 | N/A | 0 | N/A | 44 | 0.523 | 0.402 |

 > **Note:** CI intervals use 1000-iteration bootstrap resampling.
 > Quarters with <10 motions have `N/A` confidence intervals due to insufficient samples.
--- a/reports/overton_window/temporal_trajectory_figure.png
+++ b/reports/overton_window/temporal_trajectory_figure.png