refactor: delete stale files and consolidate .mindmodel structure

Deleted stale root-level Python files:
- main.py (unused 'Hello world' script)
- verify.py (unused table info script)
- scraper.py (unused MotionScraper class)
- scheduler.py (unused DataUpdateScheduler class)

Deleted duplicate .mindmodel root YAML files (subdirectory versions are more comprehensive):
- anti-patterns.yaml, architecture.yaml, conventions.yaml
- dependencies.yaml, domain.yaml, domain-glossary.yaml
- stack.yaml, tech-stack.yaml, workflows.yaml

Added comprehensive .mindmodel subdirectories:
- constraints/ (naming, db-schema, error-handling, types, etc.)
- patterns/ (api, architecture, database, python, streamlit, etc.)
- examples/ (code examples for each pattern)
- anti-patterns/, architecture/, conventions/, dependencies/, domain/, stack/

Updated ARCHITECTURE.md to reflect current codebase:
- Removed references to non-existent files
- Added missing files (explorer.py, explorer_helpers.py, pipeline/)
- Added directory structure documentation
- Updated tech stack to include scipy, sklearn, umap

Updated .gitignore:
- Added patterns for generated analysis files
- Added .worktrees/ pattern (was already in gitignore but dir was deleted)

Removed empty .worktrees/ directory
This commit is contained in:
2026-04-04 18:46:53 +02:00
parent 0308d20f12
commit f376300804
22 changed files with 3824 additions and 48 deletions
+196
View File
@@ -0,0 +1,196 @@
"""Example: TweedeKamerAPI usage - from api_client.py and actual codebase."""
from datetime import datetime, timedelta
from typing import Dict, List
# Import the API client
from api_client import TweedeKamerAPI
# =============================================================================
# Example 1: Basic API usage
# =============================================================================
def example_fetch_motions():
"""Fetch recent parliamentary motions from TweedeKamer API."""
api = TweedeKamerAPI()
# Fetch motions from last 30 days
start_date = datetime.now() - timedelta(days=30)
try:
motions = api.get_motions(start_date=start_date, limit=100)
print(f"Fetched {len(motions)} motions")
for motion in motions[:5]: # Show first 5
print(f" - {motion.get('title', 'N/A')}")
return motions
finally:
api.close()
# =============================================================================
# Example 2: Fetching with date range
# =============================================================================
def example_date_range():
"""Fetch motions from a specific date range."""
api = TweedeKamerAPI()
start = datetime(2024, 1, 1)
end = datetime(2024, 3, 31) # Q1 2024
try:
motions = api.get_motions(start_date=start, end_date=end, limit=500)
# Group by policy area
by_area = {}
for m in motions:
area = m.get("policy_area", "Onbekend")
by_area.setdefault(area, []).append(m)
for area, area_motions in sorted(by_area.items()):
print(f"{area}: {len(area_motions)} motions")
return motions
finally:
api.close()
# =============================================================================
# Example 3: Context manager usage
# =============================================================================
def example_context_manager():
"""Use API client as context manager."""
with TweedeKamerAPI() as api:
motions = api.get_motions(
start_date=datetime.now() - timedelta(days=7), limit=50
)
print(f"Fetched {len(motions)} motions this week")
return motions
# =============================================================================
# Example 4: Processing voting records
# =============================================================================
def example_process_votes():
"""Process individual voting records from API."""
api = TweedeKamerAPI()
start_date = datetime.now() - timedelta(days=7)
try:
# Get voting records directly
voting_records, besluit_meta = api._get_voting_records(
start_date=start_date, limit=1000
)
print(f"Fetched {len(voting_records)} voting records")
print(f"From {len(besluit_meta)} unique decisions")
# Count votes by party
party_votes = {}
for record in voting_records:
party = record.get("Fractie", "Onbekend")
vote = record.get("Soort", "Onbekend")
party_votes.setdefault(party, {})[vote] = (
party_votes.get(party, {}).get(vote, 0) + 1
)
for party, votes in sorted(party_votes.items()):
total = sum(votes.values())
voor = votes.get("Voor", 0)
print(f"{party}: {total} votes ({voor} voor)")
return voting_records
finally:
api.close()
# =============================================================================
# Example 5: Safe API call with fallback
# =============================================================================
def example_safe_call():
"""Make API call with safe fallback on failure."""
api = TweedeKamerAPI()
try:
# This will return [] on any error
motions = api.get_motions(
start_date=datetime.now() - timedelta(days=30), limit=100
)
if not motions:
print("No motions returned - using cached data")
# Fallback to cached/local data
from database import db
return db.get_filtered_motions(limit=10)
return motions
finally:
api.close()
# =============================================================================
# Example 6: Pagination handling
# =============================================================================
def example_pagination():
"""Understand how pagination works in the API."""
api = TweedeKamerAPI()
start_date = datetime.now() - timedelta(days=365)
# Simulate pagination
page_size = 250
total_limit = 500
all_motions = []
skip = 0
while len(all_motions) < total_limit:
print(f"Fetching page with skip={skip}...")
# In real usage, get_motions handles pagination internally
# This demonstrates what's happening under the hood
page_motions = api._fetch_page(start_date=start_date, skip=skip, top=page_size)
if not page_motions:
break
all_motions.extend(page_motions)
skip += page_size
if len(page_motions) < page_size:
break # Last page
print(f"Total fetched: {len(all_motions)} motions")
return all_motions
if __name__ == "__main__":
print("=== Basic Fetch ===")
example_fetch_motions()
print("\n=== Process Votes ===")
example_process_votes()
+191
View File
@@ -0,0 +1,191 @@
"""Example: MotionDatabase usage - from database.py and actual codebase."""
from typing import Dict, List, Optional
import duckdb
import json
from config import config
# Import the singleton instance
from database import db
# =============================================================================
# Example 1: Getting filtered motions
# =============================================================================
def example_get_filtered_motions():
"""Get controversial motions from a specific policy area."""
motions = db.get_filtered_motions(
policy_area="Klimaat",
min_margin=0.0,
max_margin=0.3, # Controversial: close margin
limit=10,
)
for motion in motions:
print(f"{motion['title']}: {motion['winning_margin']:.1%} margin")
return motions
# =============================================================================
# Example 2: Creating a voting session
# =============================================================================
def example_voting_session():
"""Create a new user session and record votes."""
# Create session for 10 motions
session_id = db.create_session(total_motions=10)
print(f"Created session: {session_id}")
# Get motions for the session
motions = db.get_filtered_motions(policy_area="Alle", limit=10)
# Record votes
for motion in motions:
# In real app, user would choose vote
vote = "Voor" # Example vote
db.record_vote(session_id=session_id, motion_id=motion["id"], vote=vote)
# Get results
results = db.get_party_results(session_id)
for party, result in sorted(results.items(), key=lambda x: -x[1]["agreement"]):
print(f"{party}: {result['agreement']:.1%} agreement")
return results
# =============================================================================
# Example 3: Working with DuckDB connections directly
# =============================================================================
def example_direct_duckdb():
"""Example of proper DuckDB connection handling."""
conn = duckdb.connect(config.DATABASE_PATH)
try:
# Get motion with votes
result = conn.execute(
"""
SELECT m.*,
JSON_EXTRACT(voting_results, '$.total_votes') as total_votes
FROM motions m
WHERE m.id = ?
""",
(123,),
).fetchone()
if result:
print(f"Motion: {result[1]}") # title is index 1
return result
finally:
conn.close()
# =============================================================================
# Example 4: Bulk operations
# =============================================================================
def example_bulk_insert():
"""Example of bulk inserting motions."""
# Sample data
motions = [
{
"title": "Motion about climate policy",
"description": "Proposal to reduce emissions",
"date": "2024-01-15",
"policy_area": "Klimaat",
"voting_results": json.dumps({"Voor": 75, "Tegen": 65}),
"winning_margin": 0.07,
"controversy_score": 0.85,
},
{
"title": "Motion about healthcare",
"description": "Increase healthcare budget",
"date": "2024-01-20",
"policy_area": "Zorg",
"voting_results": json.dumps({"Voor": 90, "Tegen": 50}),
"winning_margin": 0.29,
"controversy_score": 0.42,
},
]
conn = duckdb.connect(config.DATABASE_PATH)
try:
for motion in motions:
conn.execute(
"""
INSERT INTO motions
(title, description, date, policy_area, voting_results,
winning_margin, controversy_score)
VALUES (?, ?, ?, ?, ?, ?, ?)
""",
(
motion["title"],
motion["description"],
motion["date"],
motion["policy_area"],
motion["voting_results"],
motion["winning_margin"],
motion["controversy_score"],
),
)
conn.close()
print(f"Inserted {len(motions)} motions")
except Exception as e:
conn.close()
print(f"Error inserting motions: {e}")
# =============================================================================
# Example 5: Query with aggregation
# =============================================================================
def example_aggregation():
"""Example of aggregate queries."""
conn = duckdb.connect(config.DATABASE_PATH)
try:
# Get statistics by policy area
results = conn.execute("""
SELECT
policy_area,
COUNT(*) as motion_count,
AVG(winning_margin) as avg_margin,
AVG(controversy_score) as avg_controversy
FROM motions
WHERE policy_area IS NOT NULL
GROUP BY policy_area
ORDER BY motion_count DESC
""").fetchall()
for row in results:
print(
f"{row[0]}: {row[1]} motions, "
f"avg margin {row[2]:.1%}, "
f"controversy {row[3]:.2f}"
)
conn.close()
return results
except Exception as e:
conn.close()
return []
if __name__ == "__main__":
print("=== Filtered Motions ===")
example_get_filtered_motions()
print("\n=== Aggregation ===")
example_aggregation()
+217
View File
@@ -0,0 +1,217 @@
"""Example: Pipeline phase execution - from pipeline/run_pipeline.py and actual codebase."""
import argparse
from datetime import date, timedelta
from typing import List, Tuple
# Import pipeline modules
from pipeline.fetch_mp_metadata import fetch_mp_metadata
from pipeline.extract_mp_votes import extract_mp_votes
from pipeline.svd_pipeline import run_svd_pipeline
from pipeline.text_pipeline import run_text_pipeline
from pipeline.fusion import run_fusion
from database import MotionDatabase
# =============================================================================
# Example 1: Running full pipeline
# =============================================================================
def example_full_pipeline():
"""Run the complete data ingestion pipeline."""
# Parse arguments like CLI would
parser = argparse.ArgumentParser(description="Pipeline runner")
parser.add_argument("--db-path", default="data/motions.db")
parser.add_argument("--start-date", default=None)
parser.add_argument("--end-date", default=None)
parser.add_argument(
"--window-size", choices=["quarterly", "annual"], default="quarterly"
)
parser.add_argument("--svd-k", type=int, default=50)
args = parser.parse_args([])
# Resolve dates
end_date = date.fromisoformat(args.end_date) if args.end_date else date.today()
start_date = (
date.fromisoformat(args.start_date)
if args.start_date
else end_date - timedelta(days=730)
)
print(f"Running pipeline: {start_date}{end_date}")
print(f"Window size: {args.window_size}")
print(f"DB path: {args.db_path}")
# Initialize database
db = MotionDatabase(args.db_path)
# Phase 1: Fetch MP metadata
print("\n=== Phase 1: MP Metadata ===")
n_mp = fetch_mp_metadata(db_path=args.db_path)
print(f"Processed {n_mp} MPs")
# Phase 2: Extract MP votes
print("\n=== Phase 2: Extract Votes ===")
n_votes = extract_mp_votes(db_path=args.db_path)
print(f"Extracted {n_votes} vote records")
# Phase 3: Generate time windows
print("\n=== Phase 3: SVD Pipeline ===")
windows = generate_windows(start_date, end_date, args.window_size)
print(f"Generated {len(windows)} windows: {windows}")
# Phase 4: SVD per window
run_svd_pipeline(db, windows, args.svd_k)
print(f"Computed SVD for {len(windows)} windows")
# Phase 5: Text embeddings
print("\n=== Phase 4: Text Embeddings ===")
run_text_pipeline(args.db_path, batch_size=50)
print("Text embeddings completed")
# Phase 6: Fusion
print("\n=== Phase 5: Fusion ===")
run_fusion(args.db_path, windows)
print("Fusion completed")
print("\n=== Pipeline Complete ===")
# =============================================================================
# Example 2: Generate time windows
# =============================================================================
def generate_windows(
start: date, end: date, granularity: str
) -> List[Tuple[str, str, str]]:
"""Generate time windows for pipeline processing."""
windows = []
cursor = date(start.year, start.month, 1)
if granularity == "annual":
cursor = date(start.year, 1, 1)
while cursor <= end:
year_end = date(cursor.year, 12, 31)
w_end = min(year_end, end)
windows.append((str(cursor.year), cursor.isoformat(), w_end.isoformat()))
cursor = date(cursor.year + 1, 1, 1)
else:
# quarterly
quarter_starts = {1: 1, 2: 4, 3: 7, 4: 10}
quarter_ends = {1: 3, 2: 6, 3: 9, 4: 12}
q = (cursor.month - 1) // 3 + 1
cursor = date(cursor.year, quarter_starts[q], 1)
while cursor <= end:
q = (cursor.month - 1) // 3 + 1
import calendar
q_end_month = quarter_ends[q]
last_day = calendar.monthrange(cursor.year, q_end_month)[1]
q_end = date(cursor.year, q_end_month, last_day)
w_end = min(q_end, end)
window_id = f"{cursor.year}-Q{q}"
windows.append((window_id, cursor.isoformat(), w_end.isoformat()))
cursor = q_end + timedelta(days=1)
return windows
def example_window_generation():
"""Example of window generation."""
start = date(2023, 1, 1)
end = date(2024, 6, 30)
print("Quarterly windows:")
quarterly = generate_windows(start, end, "quarterly")
for wid, s, e in quarterly:
print(f" {wid}: {s} to {e}")
print("\nAnnual windows:")
annual = generate_windows(start, end, "annual")
for wid, s, e in annual:
print(f" {wid}: {s} to {e}")
# =============================================================================
# Example 3: Running individual phases
# =============================================================================
def example_individual_phases():
"""Run pipeline phases individually for debugging."""
db_path = "data/motions.db"
db = MotionDatabase(db_path)
# Only run MP metadata fetch
print("Fetching MP metadata...")
n = fetch_mp_metadata(db_path=db_path)
print(f" {n} MPs processed")
# Only run vote extraction
print("Extracting votes...")
n = extract_mp_votes(db_path=db_path)
print(f" {n} votes extracted")
# Only run SVD for specific window
print("Computing SVD...")
windows = [("2024-Q1", "2024-01-01", "2024-03-31")]
run_svd_pipeline(db, windows, k=50)
print(" SVD computed")
# Only run text embeddings
print("Computing embeddings...")
run_text_pipeline(db_path, batch_size=25) # Smaller batch for testing
print(" Embeddings computed")
# =============================================================================
# Example 4: Dry run
# =============================================================================
def example_dry_run():
"""Show what pipeline would do without making changes."""
print("DRY RUN - no writes will be made")
start_date = date(2024, 1, 1)
end_date = date(2024, 6, 30)
# Generate and show windows
windows = generate_windows(start_date, end_date, "quarterly")
print(f"Would process {len(windows)} windows:")
for wid, s, e in windows:
print(f" {wid}: {s} to {e}")
print("\nWould run phases:")
print(" 1. fetch_mp_metadata")
print(" 2. extract_mp_votes")
print(" 3. svd_pipeline")
print(" 4. text_pipeline")
print(" 5. fusion")
if __name__ == "__main__":
import logging
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s %(levelname)s %(name)s: %(message)s",
)
print("=== Window Generation ===")
example_window_generation()
print("\n=== Dry Run ===")
example_dry_run()
@@ -0,0 +1,316 @@
"""Example: Streamlit page patterns - from actual pages/ files."""
import streamlit as st
# =============================================================================
# Example 1: Home page (Home.py)
# =============================================================================
def render_home_page():
"""Simplified version of Home.py."""
st.set_page_config(
page_title="Motief: de stematlas",
page_icon="🗺️",
layout="centered",
initial_sidebar_state="expanded",
)
st.title("🗺️ Motief: de stematlas")
st.markdown(
"**Motief** brengt de Nederlandse Tweede Kamer in kaart op basis van "
"echte stemmingen over moties. Gebruik de Stemwijzer om te ontdekken welke "
"partij het beste bij jouw standpunten past, of verken de politieke ruimte "
"zelf in de Explorer."
)
st.divider()
col1, col2 = st.columns(2)
with col1:
st.subheader("🗳️ Stemwijzer")
st.markdown(
"Stem op echte Tweede Kamer moties en zie welke partij het "
"dichtst bij jouw keuzes staat."
)
st.page_link("pages/1_Stemwijzer.py", label="Open Stemwijzer", icon="🗳️")
with col2:
st.subheader("🔭 Politiek Explorer")
st.markdown(
"Verken het politieke kompas, partijtrajecten door de tijd, "
"en zoek vergelijkbare moties op in het archief."
)
st.page_link("pages/2_Explorer.py", label="Open Explorer", icon="🔭")
st.divider()
st.caption("Data: Tweede Kamer API · Embeddings: QWEN (via OpenRouter)")
# =============================================================================
# Example 2: Thin page wrapper (pages/1_Stemwijzer.py)
# =============================================================================
def render_stemwijzer_page():
"""Pattern: thin page that delegates to module function."""
st.set_page_config(
page_title="Stemwijzer",
page_icon="🗳️",
layout="centered",
)
# Delegate to main module
from explorer import build_mp_quiz_tab
build_mp_quiz_tab("data/motions.db")
# =============================================================================
# Example 3: Session state initialization
# =============================================================================
def init_session_state():
"""Pattern: Initialize all session state at start."""
defaults = {
"session_id": None,
"current_motion_index": 0,
"motions": [],
"show_results": False,
"user_votes": {},
}
for key, default in defaults.items():
if key not in st.session_state:
st.session_state[key] = default
# =============================================================================
# Example 4: Sidebar configuration
# =============================================================================
def render_sidebar():
"""Pattern: Sidebar for configuration."""
with st.sidebar:
st.header("Instellingen")
motion_count = st.slider(
"Aantal moties",
min_value=5,
max_value=25,
value=10,
help="Hoeveel moties wilt u beantwoorden?",
)
policy_area = st.selectbox(
"Beleidsgebied",
[
"Alle",
"Economie",
"Klimaat",
"Immigratie",
"Zorg",
"Onderwijs",
"Defensie",
"Sociale Zaken",
"Algemeen",
],
)
margin_range = st.slider(
"Controversiële moties (%)",
min_value=0,
max_value=100,
value=(0, 100),
help="Filter op hoe omstreden de moties zijn",
)
st.divider()
if st.button("Start Nieuwe Sessie", type="primary"):
return {
"motion_count": motion_count,
"policy_area": policy_area,
"margin_range": margin_range,
}
return None
# =============================================================================
# Example 5: Motion voting interface
# =============================================================================
def render_motion_vote(motion: dict, index: int, total: int):
"""Pattern: Display motion and voting buttons."""
st.subheader(f"Motie {index + 1} van {total}")
# Motion content
st.markdown(f"### {motion['title']}")
col1, col2 = st.columns([3, 1])
with col1:
if motion.get("layman_explanation"):
st.info(motion["layman_explanation"])
with st.expander("Meer details"):
st.markdown(f"**Datum:** {motion.get('date', 'Onbekend')}")
st.markdown(f"**Beleidsgebied:** {motion.get('policy_area', 'Onbekend')}")
if motion.get("description"):
st.markdown(f"**Beschrijving:** {motion['description']}")
with col2:
st.metric(
label="Winstmarge",
value=f"{motion.get('winning_margin', 0):.0%}",
delta="Omstreden" if motion.get("controversy_score", 0) > 0.5 else "Helder",
)
st.divider()
# Voting buttons
col1, col2, col3 = st.columns(3)
with col1:
st.button(
"👍 **Voor**",
on_click=on_vote,
args=(motion["id"], "Voor"),
use_container_width=True,
)
with col2:
st.button(
"👎 **Tegen**",
on_click=on_vote,
args=(motion["id"], "Tegen"),
use_container_width=True,
)
with col3:
st.button(
"🤔 **Onthouden**",
on_click=on_vote,
args=(motion["id"], "Onthouden"),
use_container_width=True,
)
def on_vote(motion_id: int, vote: str):
"""Callback when user votes."""
# Record vote
from database import db
db.record_vote(
session_id=st.session_state.session_id, motion_id=motion_id, vote=vote
)
# Update session state
st.session_state.user_votes[motion_id] = vote
# Move to next or show results
if st.session_state.current_motion_index < len(st.session_state.motions) - 1:
st.session_state.current_motion_index += 1
else:
st.session_state.show_results = True
st.rerun()
# =============================================================================
# Example 6: Results display
# =============================================================================
def render_results():
"""Pattern: Display voting results."""
from database import db
st.header("📊 Uw Resultaten")
# Get party results
results = db.get_party_results(st.session_state.session_id)
if not results:
st.warning("Geen resultaten beschikbaar")
return
# Sort by agreement
sorted_results = sorted(
results.items(), key=lambda x: x[1].get("agreement_percentage", 0), reverse=True
)
# Display top match
if sorted_results:
top_party, top_data = sorted_results[0]
st.success(
f"**Uw beste match:** {top_party} ({top_data.get('agreement_percentage', 0):.0%} overeenstemming)"
)
st.divider()
# Show all parties
for party, data in sorted_results:
agreement = data.get("agreement_percentage", 0)
col1, col2 = st.columns([3, 1])
with col1:
st.markdown(f"**{party}**")
st.progress(agreement, text=f"{agreement:.0%}")
with col2:
st.metric("Overeenstemming", f"{agreement:.0%}")
# Detailed breakdown
with st.expander("Details per motie"):
for motion in st.session_state.motions:
user_vote = st.session_state.user_votes.get(motion["id"], "?")
st.markdown(f"- **{motion['title']}**: U={user_vote}")
# =============================================================================
# Example 7: Tabs layout
# =============================================================================
def render_tabs_example():
"""Pattern: Use tabs for organizing content."""
tab1, tab2, tab3 = st.tabs(["Compass", "Trajectories", "Zoeken"])
with tab1:
st.subheader("Politiek Kompas")
st.write("Visualiseer partijposities in 2D ruimte")
# Add compass chart...
with tab2:
st.subheader("Partij Trajectories")
st.write("Bekijk hoe partijen door de tijd bewegen")
# Add trajectory chart...
with tab3:
st.subheader("Zoek Moties")
query = st.text_input("Zoekterm")
if query:
# Search functionality...
st.write(f"Zoeken naar: {query}")
if __name__ == "__main__":
# Demo rendering
init_session_state()
st.write("Streamlit page structure example")