feat(pipeline): implement parliamentary embedding pipeline MVP

- Add 4 migration files: mp_votes, mp_metadata, svd_vectors, fused_embeddings
- Extend database.py with 5 new helper methods and table init
- Add pipeline/ package: extract_mp_votes, fetch_mp_metadata, text_pipeline,
  svd_pipeline (with Procrustes alignment), fusion
- Add full test suite (17 tests) covering all pipeline modules and migrations
- Fix Procrustes alignment bug: scipy scale is a norm value, not a multiplier
- Fix DuckDB date type handling in test assertions (datetime.date vs string)
- Remove duckdb.py shim; tests now run against real duckdb + scipy via uv

Ref: thoughts/shared/plans/2026-03-21-parliamentary-embedding-pipeline-plan.md

tests/__init__.py

@@ -0,0 +1 @@
+"""Make the tests directory a package so test helpers can be imported."""

tests/conftest.py

@@ -0,0 +1,63 @@
+import tempfile
+import pytest
+
+# Load test fixtures from the utils package so pytest can discover them.
+pytest_plugins = ["tests.utils.migration_fixtures"]
+
+
+@pytest.fixture
+def tmp_duckdb_path(tmp_path):
+    p = tmp_path / "test.db"
+    return str(p)
+
+
+@pytest.fixture
+def tmp_duckdb_conn(tmp_duckdb_path):
+    # Import duckdb lazily so running pytest doesn't fail on machines
+    # where duckdb is not installed (CI / contributor machines that don't
+    # need the duckdb-based fixtures). If duckdb is missing, skip this
+    # fixture at runtime when it's requested.
+    try:
+        import duckdb
+    except Exception:
+        pytest.skip("duckdb not installed, skipping duckdb fixtures")
+
+    conn = duckdb.connect(database=tmp_duckdb_path)
+    yield conn
+    try:
+        conn.close()
+    except Exception:
+        pass
+
+
+@pytest.fixture
+def monkeypatch_ai_provider(monkeypatch):
+    """Patch ai_provider.get_embedding to return deterministic 16-dim vector."""
+    import ai_provider
+
+    fake = [0.01] * 16
+    monkeypatch.setattr(ai_provider, "get_embedding", lambda text, model=None: fake)
+    return fake
+
+
+@pytest.fixture
+def mock_odata_client(monkeypatch):
+    """
+    Patch requests.Session.get for OData calls.
+    Returns a configurable mock — set mock_odata_client.response to override.
+    """
+    import requests
+    from unittest.mock import MagicMock
+
+    mock_response = MagicMock()
+    mock_response.raise_for_status.return_value = None
+    mock_response.json.return_value = {"value": []}
+
+    class MockSession:
+        response = mock_response
+
+        def get(self, *args, **kwargs):
+            return self.response
+
+    monkeypatch.setattr(requests, "Session", MockSession)
+    return mock_response

tests/fixtures/__init__.py

@@ -0,0 +1 @@
+"""Fixtures package for tests."""

tests/fixtures/sample_voting_results.json

@@ -0,0 +1,40 @@
+[
+  {
+    "motion_id": 1,
+    "date": "2024-01-15",
+    "voting_results": {
+      "VVD": "voor",
+      "PvdA": "tegen",
+      "CDA": "voor",
+      "D66": "voor",
+      "Wilders, G.": "voor",
+      "Yesilgöz-Zegerius, D.": "voor",
+      "Jetten, R.A.A.": "voor"
+    }
+  },
+  {
+    "motion_id": 2,
+    "date": "2024-02-10",
+    "voting_results": {
+      "VVD": "tegen",
+      "PvdA": "voor",
+      "CDA": "afwezig",
+      "D66": "voor",
+      "Wilders, G.": "tegen",
+      "Yesilgöz-Zegerius, D.": "tegen",
+      "Ploumen, L.J.": "voor"
+    }
+  },
+  {
+    "motion_id": 3,
+    "date": "2024-03-05",
+    "voting_results": {
+      "VVD": "voor",
+      "SP": "tegen",
+      "GroenLinks": "voor",
+      "PVV": "voor",
+      "Van der Plas, C.": "voor",
+      "Klever, N.C.": "voor"
+    }
+  }
+]

tests/integration/test_pipeline_end_to_end.py

@@ -0,0 +1,87 @@
+import json
+import os
+import numpy as np
+import pytest
+
+# duckdb is an optional dependency in some environments; skip test if not available
+duckdb = pytest.importorskip("duckdb")
+
+
+def test_pipeline_end_to_end(tmp_path, monkeypatch):
+    # ensure determinism for any random embedding generation
+    np.random.seed(0)
+
+    # prepare temp db
+    db_path = str(tmp_path / "motions.db")
+
+    # create the minimal MotionDatabase schema using existing code where possible
+    from database import MotionDatabase
+
+    db = MotionDatabase(db_path)
+
+    # create embeddings table (migration would normally do this)
+    conn = duckdb.connect(db.db_path)
+    conn.execute("CREATE SEQUENCE IF NOT EXISTS embeddings_id_seq START 1")
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS embeddings (id INTEGER PRIMARY KEY DEFAULT nextval('embeddings_id_seq'), motion_id INTEGER, model TEXT, vector JSON, created_at TIMESTAMP)"
+    )
+
+    # insert three motions
+    conn.execute(
+        "INSERT INTO motions (title, description, url, layman_explanation) VALUES (?, ?, ?, ?)",
+        ("t1", "d1", "u1", "ex1"),
+    )
+    conn.execute(
+        "INSERT INTO motions (title, description, url, layman_explanation) VALUES (?, ?, ?, ?)",
+        ("t2", "d2", "u2", "ex2"),
+    )
+    conn.execute(
+        "INSERT INTO motions (title, description, url, layman_explanation) VALUES (?, ?, ?, ?)",
+        ("t3", "d3", "u3", "ex3"),
+    )
+
+    # fetch ids
+    rows = conn.execute("SELECT id FROM motions ORDER BY id").fetchall()
+    ids = [r[0] for r in rows]
+
+    # insert existing embedding for first motion
+    vec = json.dumps([0.1] * 16)
+    conn.execute(
+        "INSERT INTO embeddings (motion_id, model, vector) VALUES (?, ?, ?)",
+        (ids[0], "test-model", vec),
+    )
+
+    conn.close()
+
+    # monkeypatch ai_provider.get_embedding to deterministic vector
+    import ai_provider
+
+    def fake_get_embedding(text, model=None):
+        # produce a deterministic vector based on seeded numpy
+        return list(np.random.rand(16))
+
+    monkeypatch.setattr("ai_provider.get_embedding", fake_get_embedding)
+
+    # run ensure_text_embeddings
+    from pipeline.text_pipeline import ensure_text_embeddings
+
+    stored, skipped_existing, skipped_no_text, errors = ensure_text_embeddings(
+        db_path=db_path, model="test-model"
+    )
+
+    assert stored == 2
+    assert skipped_existing == 1
+    assert skipped_no_text == 0
+    assert errors == 0
+
+    # verify stored vectors length
+    conn = duckdb.connect(db.db_path)
+    rows = conn.execute(
+        "SELECT vector FROM embeddings WHERE model = ? ORDER BY motion_id",
+        ("test-model",),
+    ).fetchall()
+    conn.close()
+    assert len(rows) == 3
+    for r in rows:
+        v = json.loads(r[0])
+        assert len(v) == 16

tests/migrations/test_2026_03_22_add_audit_events.py

@@ -0,0 +1,58 @@
+import os
+import pathlib
+import sqlite3
+import re
+import pytest
+
+
+def test_migration_file_exists_and_name():
+    migrations_dir = pathlib.Path("migrations")
+    expected_name = "2026-03-22-add-audit-events.sql"
+    migration_path = migrations_dir / expected_name
+
+    # File must exist
+    assert migration_path.exists(), f"Migration file {migration_path} does not exist"
+
+    # Name sanity check
+    assert migration_path.name == expected_name
+
+
+def _strip_sql_comments(sql_text: str) -> str:
+    # Remove SQL single-line comments -- ... and C-style /* ... */
+    # Use multiline-aware single-line removal for safety.
+    no_single = re.sub(r"--.*?$", "", sql_text, flags=re.MULTILINE)
+    no_block = re.sub(r"/\*.*?\*/", "", no_single, flags=re.DOTALL)
+    return no_block.strip()
+
+
+def test_optional_apply_sql_if_db_available():
+    """
+    If TEST_DB_URL is provided, attempt to apply the SQL.
+
+    For safety this test will skip applying when the SQL is empty or commented out.
+    Only sqlite URLs (sqlite:///path/to/db) are attempted here to avoid adding
+    extra dependencies; other URL schemes will cause the test to be skipped.
+    """
+    db_url = os.environ.get("TEST_DB_URL")
+    if not db_url:
+        pytest.skip("TEST_DB_URL not set - skipping DB application")
+
+    migration_path = pathlib.Path("migrations") / "2026-03-22-add-audit-events.sql"
+    sql = migration_path.read_text(encoding="utf8")
+    stripped = _strip_sql_comments(sql)
+    if not stripped:
+        pytest.skip("Migration SQL is empty or commented out - skipping application")
+
+    # Only handle sqlite URLs here
+    if db_url.startswith("sqlite:///"):
+        db_path = db_url.replace("sqlite:///", "", 1)
+        try:
+            conn = sqlite3.connect(db_path)
+            try:
+                conn.executescript(sql)
+            finally:
+                conn.close()
+        except Exception as e:
+            pytest.skip(f"Could not apply SQL to sqlite DB: {e}")
+    else:
+        pytest.skip(f"TEST_DB_URL set but scheme not supported by this test: {db_url}")

tests/migrations/test_2026_03_22_add_similarity_cache.py

@@ -0,0 +1,85 @@
+import os
+import re
+import pathlib
+import pytest
+# small migration filename/header tests; keep imports minimal
+
+
+MIGRATION_FILENAME = "2026-03-22-add-similarity-cache.sql"
+MIGRATION_PATH = pathlib.Path("migrations") / MIGRATION_FILENAME
+
+
+def _strip_sql_comments(sql: str) -> str:
+    """Remove SQL single-line (-- ...) and C-style (/* ... */) comments.
+
+    This is a best-effort stripper sufficient for the test's purpose.
+    """
+    # remove block comments
+    sql = re.sub(r"/\*.*?\*/", "", sql, flags=re.S)
+    # remove line comments
+    sql = re.sub(r"--.*?$", "", sql, flags=re.M)
+    return sql.strip()
+
+
+def test_migration_file_exists_and_header():
+    # file must exist
+    assert MIGRATION_PATH.exists(), f"Migration file {MIGRATION_PATH} not found"
+
+    text = MIGRATION_PATH.read_text(encoding="utf8")
+
+    # header should reference the filename and purpose
+    assert MIGRATION_FILENAME in text.splitlines()[0], (
+        "First line should include the filename"
+    )
+    assert "similarity" in text.lower(), "Header should mention similarity"
+
+
+def test_optional_apply_migration_safe():
+    # If TEST_DB_URL is set, try to apply the SQL only if it contains non-comment statements.
+    db_url = os.environ.get("TEST_DB_URL")
+    sql = MIGRATION_PATH.read_text(encoding="utf8")
+    stripped = _strip_sql_comments(sql)
+
+    # If there is no DB url, consider this a filename/header validation test only.
+    if not db_url:
+        pytest.skip("TEST_DB_URL not set; skipping DB apply step")
+
+    # If the SQL is empty (only comments), nothing to apply — test passes.
+    if not stripped:
+        pytest.skip("Migration contains no executable SQL; nothing to apply")
+
+    # Otherwise attempt to execute the SQL. Be conservative: if drivers are missing or
+    # connection fails, skip the test rather than failing CI. Only unexpected errors
+    # during execution should fail the test.
+    try:
+        if db_url.startswith("sqlite:"):
+            import sqlite3
+
+            # sqlite URL might be sqlite:///path or sqlite:///:memory:
+            path = db_url.split("sqlite:", 1)[1]
+            # normalize prefixes like ///
+            path = path.lstrip("/") or ":memory:"
+            conn = sqlite3.connect(path)
+            try:
+                conn.executescript(sql)
+            finally:
+                conn.close()
+        elif db_url.startswith("postgresql:") or db_url.startswith("postgres:"):
+            try:
+                import psycopg2
+            except Exception as e:  # pragma: no cover - driver may be absent in CI
+                pytest.skip(f"psycopg2 not available: {e}")
+
+            # psycopg2 accepts a DSN; rely on that here.
+            conn = psycopg2.connect(db_url)
+            try:
+                cur = conn.cursor()
+                cur.execute(sql)
+                conn.commit()
+            finally:
+                conn.close()
+        else:
+            pytest.skip(f"DB URL scheme not supported by this test: {db_url}")
+    except Exception as exc:
+        # Unexpected error while applying SQL should fail the test.
+        raise

tests/migrations/test_migration_fixtures_smoke.py

@@ -0,0 +1,29 @@
+"""Smoke test for the migration test_db fixture.
+
+This test imports the `test_db` fixture and asserts expected behavior in two
+cases:
+
+- If the environment variable TEST_DB_URL is not set, the fixture should yield
+  None.
+- If TEST_DB_URL is set, the fixture should yield a connection-like object
+  (we check for an object with a `cursor` attribute or the sqlite3 connection
+  type).
+"""
+
+import os
+import types
+
+import pytest
+
+
+def test_migration_fixture_smoke(test_db):
+    """Smoke test ensuring the test_db fixture yields expected values."""
+    url = os.environ.get("TEST_DB_URL")
+    if not url:
+        assert test_db is None
+    else:
+        # For sqlite we expect a sqlite3.Connection which has a 'cursor'
+        # method. Be permissive and accept any object with a 'cursor'
+        # attribute or callable.
+        assert test_db is not None
+        assert hasattr(test_db, "cursor") or hasattr(test_db, "execute")

tests/test_ai_provider.py

@@ -0,0 +1,49 @@
+import os
+import types
+
+import pytest
+
+import ai_provider
+
+
+class DummyResponse:
+    def __init__(self, status_code=200, json_data=None):
+        self.status_code = status_code
+        self._json = json_data or {}
+
+    def json(self):
+        return self._json
+
+
+def test_get_embedding_success(monkeypatch):
+    fake = DummyResponse(json_data={"data": [{"embedding": [0.1, 0.2, 0.3]}]})
+
+    def fake_post(url, json, headers, timeout):
+        return fake
+
+    monkeypatch.setenv("OPENROUTER_API_KEY", "sk-test")
+    monkeypatch.setattr("requests.post", fake_post)
+
+    emb = ai_provider.get_embedding("hello world")
+    assert emb == [0.1, 0.2, 0.3]
+
+
+def test_chat_completion_success(monkeypatch):
+    fake = DummyResponse(json_data={"choices": [{"message": {"content": "summary"}}]})
+
+    def fake_post(url, json, headers, timeout):
+        return fake
+
+    monkeypatch.setenv("OPENROUTER_API_KEY", "sk-test")
+    monkeypatch.setattr("requests.post", fake_post)
+
+    out = ai_provider.chat_completion([{"role": "user", "content": "hi"}])
+    assert out == "summary"
+
+
+def test_missing_api_key_raises(monkeypatch):
+    # Ensure env var is not set
+    monkeypatch.delenv("OPENROUTER_API_KEY", raising=False)
+
+    with pytest.raises(ai_provider.ProviderError):
+        ai_provider.get_embedding("x")

tests/test_extract_mp_votes.py

@@ -0,0 +1,74 @@
+import json
+import duckdb
+import logging
+
+from pipeline.extract_mp_votes import extract_mp_votes
+from database import MotionDatabase
+
+
+def test_extract_mp_votes(tmp_path):
+    db_file = tmp_path / "test.db"
+
+    # Initialize database
+    mdb = MotionDatabase(db_path=str(db_file))
+
+    # Load fixture
+    fixture_path = "tests/fixtures/sample_voting_results.json"
+    with open(fixture_path, "r") as fh:
+        fixtures = json.load(fh)
+
+    # Insert motions into motions table
+    conn = duckdb.connect(str(db_file))
+    try:
+        for item in fixtures:
+            motion_id = item.get("motion_id")
+            date = item.get("date")
+            voting_results = item.get("voting_results")
+
+            conn.execute(
+                """
+                INSERT INTO motions (id, title, description, date, policy_area, voting_results, winning_margin, url)
+                VALUES (?, ?, ?, ?, ?, ?, ?, ?)
+                """,
+                (
+                    motion_id,
+                    f"Test Motion {motion_id}",
+                    "",
+                    date,
+                    "Test",
+                    json.dumps(voting_results),
+                    0.5,
+                    f"http://example/{motion_id}",
+                ),
+            )
+    finally:
+        conn.close()
+
+    # Run extraction
+    res = extract_mp_votes(db_path=str(db_file))
+
+    # Expected MP rows: count keys that contain a comma in fixtures
+    expected_mp_count = 0
+    for item in fixtures:
+        for k in item.get("voting_results", {}).keys():
+            if "," in k:
+                expected_mp_count += 1
+
+    assert res["mp_rows_inserted"] == expected_mp_count
+    assert res["motions_skipped"] == 0
+
+    # Verify mp_votes table contains only rows with comma in mp_name and count matches
+    conn = duckdb.connect(str(db_file))
+    try:
+        rows = conn.execute("SELECT mp_name FROM mp_votes").fetchall()
+    finally:
+        conn.close()
+
+    assert len(rows) == expected_mp_count
+    for (mp_name,) in rows:
+        assert "," in mp_name
+
+    # Running again should be idempotent: no new mp rows, motions_skipped > 0
+    res2 = extract_mp_votes(db_path=str(db_file))
+    assert res2["mp_rows_inserted"] == 0
+    assert res2["motions_skipped"] > 0

tests/test_fetch_mp_metadata.py

@@ -0,0 +1,103 @@
+import json
+import requests
+import types
+import pytest
+
+try:
+    import duckdb
+except Exception:
+    pytest.skip(
+        "duckdb not installed, skipping fetch_mp_metadata tests",
+        allow_module_level=True,
+    )
+
+from pipeline.fetch_mp_metadata import fetch_mp_metadata, normalize_mp_name
+
+
+class MockResponse:
+    def __init__(self, data, status_code=200):
+        self._data = data
+        self.status_code = status_code
+
+    def raise_for_status(self):
+        if not (200 <= self.status_code < 300):
+            raise requests.HTTPError(f"status {self.status_code}")
+
+    def json(self):
+        return self._data
+
+
+class MockSession:
+    def __init__(self, response):
+        self._response = response
+
+    def get(self, url):
+        return self._response
+
+
+def test_fetch_mp_metadata_idempotent(tmp_path, monkeypatch):
+    # Prepare canned OData response with two FractieZetelPersoon records
+    data = {
+        "value": [
+            {
+                "Persoon": {
+                    "Achternaam": "Yesilgöz-Zegerius",
+                    "Initialen": "D.",
+                    "Tussenvoegsel": None,
+                    "Id": "guid-1",
+                },
+                "FractieZetel": {"Fractie": {"NaamNL": "VVD"}},
+                "Van": "2023-01-01",
+                "TotEnMet": None,
+            },
+            {
+                "Persoon": {
+                    "Achternaam": "Plas",
+                    "Initialen": "C.",
+                    "Tussenvoegsel": "van der",
+                    "Id": "guid-2",
+                },
+                "FractieZetel": {"Fractie": {"NaamNL": "BBB"}},
+                "Van": "2023-06-01",
+                "TotEnMet": "2024-01-01",
+            },
+        ]
+    }
+
+    mock_resp = MockResponse(data)
+    mock_session = MockSession(mock_resp)
+
+    # Patch requests.Session to return our mock session
+    monkeypatch.setattr(requests, "Session", lambda: mock_session)
+
+    db_path = str(tmp_path / "test.db")
+
+    # First run
+    count = fetch_mp_metadata(db_path=db_path, odata_url="http://example/odata")
+    assert count == 2
+
+    # Verify DB contents
+    conn = duckdb.connect(db_path)
+    rows = conn.execute(
+        "SELECT mp_name, party, van, tot_en_met, persoon_id FROM mp_metadata ORDER BY mp_name"
+    ).fetchall()
+    conn.close()
+
+    assert len(rows) == 2
+
+    # Check normalized names
+    assert rows[0][0] == normalize_mp_name("Plas", "C.", "van der")
+    assert rows[0][1] == "BBB"
+    assert str(rows[0][2]) == "2023-06-01"
+    assert str(rows[0][3]) == "2024-01-01"
+    assert rows[0][4] == "guid-2"
+
+    assert rows[1][0] == normalize_mp_name("Yesilgöz-Zegerius", "D.", None)
+    assert rows[1][1] == "VVD"
+    assert str(rows[1][2]) == "2023-01-01"
+    assert rows[1][3] == None
+    assert rows[1][4] == "guid-1"
+
+    # Run again to assert idempotence (no exception and same count processed)
+    count2 = fetch_mp_metadata(db_path=db_path, odata_url="http://example/odata")
+    assert count2 == 2

tests/test_fusion.py

@@ -0,0 +1,79 @@
+import json
+
+import duckdb
+import pytest
+
+from database import MotionDatabase
+
+
+def test_fuse_for_window(tmp_path):
+    db_path = str(tmp_path / "motions.db")
+
+    # Create MotionDatabase (this will initialize schema except embeddings)
+    db = MotionDatabase(db_path=db_path)
+
+    # Create embeddings table (migration not run by MotionDatabase)
+    conn = duckdb.connect(db_path)
+    conn.execute("CREATE SEQUENCE IF NOT EXISTS embeddings_id_seq START 1")
+    conn.execute(
+        """
+        CREATE TABLE IF NOT EXISTS embeddings (
+            id INTEGER DEFAULT nextval('embeddings_id_seq'),
+            motion_id INTEGER NOT NULL,
+            model TEXT NOT NULL,
+            vector JSON NOT NULL,
+            created_at TIMESTAMP DEFAULT current_timestamp,
+            PRIMARY KEY (id)
+        )
+        """
+    )
+    conn.close()
+
+    # Insert 3 synthetic SVD vectors (k=4)
+    svd1 = [0.1, 0.2, 0.3, 0.4]
+    svd2 = [0.2, 0.1, 0.0, -0.1]
+    svd3 = [0.9, 0.8, 0.7, 0.6]
+
+    db.store_svd_vector("2024-Q1", "motion", "1", svd1)
+    db.store_svd_vector("2024-Q1", "motion", "2", svd2)
+    db.store_svd_vector("2024-Q1", "motion", "3", svd3)
+
+    # Insert text embeddings for motions 1 and 2 (16 dims)
+    text1 = [float(i) / 100.0 for i in range(16)]
+    text2 = [float(i) / 50.0 for i in range(16)]
+
+    conn = duckdb.connect(db_path)
+    conn.execute(
+        "INSERT INTO embeddings (motion_id, model, vector, created_at) VALUES (?, ?, ?, current_timestamp)",
+        (1, "text-model-1", json.dumps(text1)),
+    )
+    conn.execute(
+        "INSERT INTO embeddings (motion_id, model, vector, created_at) VALUES (?, ?, ?, current_timestamp)",
+        (2, "text-model-1", json.dumps(text2)),
+    )
+    conn.close()
+
+    # Import fuse function here to ensure module available
+    from pipeline.fusion import fuse_for_window
+
+    result = fuse_for_window("2024-Q1", db_path=db_path)
+
+    assert result["inserted"] == 2
+    assert result["skipped_missing_text"] == 1
+
+    # Verify fused embeddings stored
+    conn = duckdb.connect(db_path)
+    rows = conn.execute(
+        "SELECT motion_id, vector, svd_dims, text_dims FROM fused_embeddings WHERE window_id = ?",
+        ("2024-Q1",),
+    ).fetchall()
+    conn.close()
+
+    # Expect two rows for motions 1 and 2
+    assert len(rows) == 2
+
+    for motion_id, vector_json, svd_dims, text_dims in rows:
+        vec = json.loads(vector_json)
+        assert svd_dims == 4
+        assert text_dims == 16
+        assert len(vec) == 20

tests/test_migration_embeddings.py

@@ -0,0 +1,31 @@
+import os
+import pytest
+
+
+def test_embeddings_migration_creates_table(tmp_path):
+    try:
+        import duckdb
+    except ImportError:
+        pytest.skip("duckdb is not installed")
+
+    db_file = str(tmp_path / "migrations_test.db")
+    conn = duckdb.connect(database=db_file)
+    try:
+        sql = open("migrations/2026-03-19-add-embeddings.sql", "r").read()
+        conn.execute(sql)
+        # Use sequence to set id if present, otherwise provide explicit id
+        try:
+            next_id = conn.execute("SELECT nextval('embeddings_id_seq')").fetchone()[0]
+        except Exception:
+            next_id = 1
+        conn.execute(
+            "INSERT INTO embeddings (id, motion_id, model, vector) VALUES (?, ?, ?, ?)",
+            (next_id, 1, "m1", "[0.1, 0.2]"),
+        )
+        res = conn.execute(
+            "SELECT motion_id, model FROM embeddings WHERE motion_id = 1"
+        ).fetchall()
+        assert len(res) == 1
+        assert res[0][1] == "m1"
+    finally:
+        conn.close()

tests/test_migration_pipeline_tables.py

@@ -0,0 +1,219 @@
+from pathlib import Path
+
+try:
+    import duckdb
+
+    DB_BACKEND = "duckdb"
+except Exception:
+    import sqlite3
+
+    DB_BACKEND = "sqlite3"
+
+
+MIGRATIONS = [
+    (
+        "migrations/2026_03_21__create_mp_votes.sql",
+        "mp_votes",
+        [
+            "id",
+            "motion_id",
+            "mp_name",
+            "party",
+            "vote",
+            "date",
+            "created_at",
+        ],
+    ),
+    (
+        "migrations/2026_03_21__create_mp_metadata.sql",
+        "mp_metadata",
+        [
+            "mp_name",
+            "party",
+            "van",
+            "tot_en_met",
+            "persoon_id",
+        ],
+    ),
+    (
+        "migrations/2026_03_21__create_svd_vectors.sql",
+        "svd_vectors",
+        [
+            "id",
+            "window_id",
+            "entity_type",
+            "entity_id",
+            "vector",
+            "model",
+            "created_at",
+        ],
+    ),
+    (
+        "migrations/2026_03_21__create_fused_embeddings.sql",
+        "fused_embeddings",
+        [
+            "id",
+            "motion_id",
+            "window_id",
+            "vector",
+            "svd_dims",
+            "text_dims",
+            "created_at",
+        ],
+    ),
+]
+
+
+def test_run_migrations_and_tables(tmp_path):
+    db_path = tmp_path / "test.db"
+    if DB_BACKEND == "duckdb":
+        conn = duckdb.connect(str(db_path))
+    else:
+        conn = sqlite3.connect(str(db_path))
+
+    for sql_path, table_name, expected_cols in MIGRATIONS:
+        p = Path(sql_path)
+        assert p.exists(), f"Migration file {sql_path} must exist"
+        sql = p.read_text()
+
+        # If using sqlite3, transform SQL to be sqlite compatible
+        if DB_BACKEND == "sqlite3":
+            # remove CREATE SEQUENCE lines
+            lines = [
+                l
+                for l in sql.splitlines()
+                if not l.strip().upper().startswith("CREATE SEQUENCE")
+            ]
+            sql2 = "\n".join(lines)
+            # remove DEFAULT nextval(...) occurrences
+            import re
+
+            sql2 = re.sub(
+                r"DEFAULT\s+nextval\('[^']+'\)", "", sql2, flags=re.IGNORECASE
+            )
+            # replace JSON type with TEXT
+            sql2 = re.sub(r"\bJSON\b", "TEXT", sql2, flags=re.IGNORECASE)
+            # execute as script (multiple statements)
+            conn.executescript(sql2)
+        else:
+            # execute migration SQL
+            conn.execute(sql)
+
+        # check columns via pragma
+        if DB_BACKEND == "duckdb":
+            rows = conn.execute(f"PRAGMA table_info('{table_name}')").fetchall()
+            col_names = [r[1] for r in rows]
+        else:
+            cur = conn.execute(f"PRAGMA table_info('{table_name}')")
+            rows = cur.fetchall()
+            col_names = [r[1] for r in rows]
+
+        for col in expected_cols:
+            assert col in col_names, (
+                f"Column {col} missing in table {table_name}, got {col_names}"
+            )
+
+        # perform a simple insert + select to validate basic round-trip
+        if table_name == "mp_votes":
+            if DB_BACKEND == "duckdb":
+                conn.execute(
+                    "INSERT INTO mp_votes (motion_id, mp_name, party, vote, date) VALUES (1, 'Jane Doe', 'PartyX', 'Yea', '2026-03-21')"
+                )
+                res = conn.execute(
+                    "SELECT motion_id, mp_name, party, vote, date FROM mp_votes WHERE motion_id=1"
+                ).fetchone()
+                # DuckDB returns datetime.date for DATE columns; normalise to string
+                assert (
+                    res[:4] == (1, "Jane Doe", "PartyX", "Yea")
+                    and str(res[4]) == "2026-03-21"
+                )
+            else:
+                # sqlite: id has no default after transformation, provide id explicitly
+                conn.execute(
+                    "INSERT INTO mp_votes (id, motion_id, mp_name, party, vote, date) VALUES (1, 1, 'Jane Doe', 'PartyX', 'Yea', '2026-03-21')"
+                )
+                res = conn.execute(
+                    "SELECT motion_id, mp_name, party, vote, date FROM mp_votes WHERE id=1"
+                ).fetchone()
+                assert res == (1, "Jane Doe", "PartyX", "Yea", "2026-03-21")
+
+        elif table_name == "mp_metadata":
+            conn.execute(
+                "INSERT INTO mp_metadata (mp_name, party, van, tot_en_met, persoon_id) VALUES ('Jane Doe', 'PartyX', '2020-01-01', '2024-12-31', 'pid-123')"
+            )
+            res = conn.execute(
+                "SELECT mp_name, party, van, tot_en_met, persoon_id FROM mp_metadata WHERE mp_name='Jane Doe'"
+            ).fetchone()
+            # DuckDB returns datetime.date for DATE columns; normalise to string
+            assert (
+                res[0] == "Jane Doe"
+                and res[1] == "PartyX"
+                and str(res[2]) == "2020-01-01"
+                and str(res[3]) == "2024-12-31"
+                and res[4] == "pid-123"
+            )
+
+        elif table_name == "svd_vectors":
+            # JSON value as text
+            if DB_BACKEND == "duckdb":
+                conn.execute(
+                    "INSERT INTO svd_vectors (window_id, entity_type, entity_id, vector, model) VALUES ('w1', 'typeA', 'e1', '[1,2,3]', 'm1')"
+                )
+                res = conn.execute(
+                    "SELECT window_id, entity_type, entity_id, vector, model FROM svd_vectors WHERE window_id='w1'"
+                ).fetchone()
+                # Note: DuckDB may return the JSON column as string; compare string form
+                assert (
+                    res[0] == "w1"
+                    and res[1] == "typeA"
+                    and res[2] == "e1"
+                    and (str(res[3]) == "[1,2,3]" or res[3] == "[1,2,3]")
+                    and res[4] == "m1"
+                )
+            else:
+                # sqlite: provide id explicitly
+                conn.execute(
+                    "INSERT INTO svd_vectors (id, window_id, entity_type, entity_id, vector, model) VALUES (1, 'w1', 'typeA', 'e1', '[1,2,3]', 'm1')"
+                )
+                res = conn.execute(
+                    "SELECT window_id, entity_type, entity_id, vector, model FROM svd_vectors WHERE id=1"
+                ).fetchone()
+                assert (
+                    res[0] == "w1"
+                    and res[1] == "typeA"
+                    and res[2] == "e1"
+                    and str(res[3]) == "[1,2,3]"
+                    and res[4] == "m1"
+                )
+
+        elif table_name == "fused_embeddings":
+            if DB_BACKEND == "duckdb":
+                conn.execute(
+                    "INSERT INTO fused_embeddings (motion_id, window_id, vector, svd_dims, text_dims) VALUES (2, 'w2', '[0.1,0.2]', 16, 128)"
+                )
+                res = conn.execute(
+                    "SELECT motion_id, window_id, vector, svd_dims, text_dims FROM fused_embeddings WHERE motion_id=2"
+                ).fetchone()
+                assert (
+                    res[0] == 2
+                    and res[1] == "w2"
+                    and (str(res[2]) == "[0.1,0.2]" or res[2] == "[0.1,0.2]")
+                    and res[3] == 16
+                    and res[4] == 128
+                )
+            else:
+                conn.execute(
+                    "INSERT INTO fused_embeddings (id, motion_id, window_id, vector, svd_dims, text_dims) VALUES (1, 2, 'w2', '[0.1,0.2]', 16, 128)"
+                )
+                res = conn.execute(
+                    "SELECT motion_id, window_id, vector, svd_dims, text_dims FROM fused_embeddings WHERE id=1"
+                ).fetchone()
+                assert (
+                    res[0] == 2
+                    and res[1] == "w2"
+                    and str(res[2]) == "[0.1,0.2]"
+                    and res[3] == 16
+                    and res[4] == 128
+                )
+
+    conn.close()

tests/test_pyproject_deps.py

@@ -0,0 +1,5 @@
+def test_scientific_deps_present():
+    content = open("pyproject.toml").read()
+    assert "scipy" in content
+    assert "umap-learn" in content
+    assert "plotly" in content

tests/test_svd_pipeline.py

@@ -0,0 +1,63 @@
+import json
+import numpy as np
+import pytest
+
+from database import db as motion_db
+from pipeline.svd_pipeline import (
+    _safe_k,
+    _build_vote_matrix,
+    _procrustes_align,
+    run_svd_for_window,
+)
+
+
+def test_safe_k_and_build_and_run(tmp_path):
+    np.random.seed(0)
+    # reset DB file for test
+    db_path = tmp_path / "test.db"
+    # point the MotionDatabase to this test DB
+    motion_db.db_path = str(db_path)
+    motion_db._init_database()
+
+    # Create synthetic dataset: 5 MPs x 6 motions
+    mps = [f"MP_{i}" for i in range(5)]
+    motions = list(range(100, 106))
+    dates = ["2020-01-0" + str(i + 1) for i in range(6)]
+
+    votes = ["Voor", "Tegen", "Geen stem"]
+
+    # insert votes: fill full matrix using MotionDatabase helper
+    for j, motion_id in enumerate(motions):
+        for i, mp in enumerate(mps):
+            vote = votes[(i + j) % len(votes)]
+            motion_db.insert_mp_vote(motion_id, mp, vote, date=dates[j])
+
+    mat, mp_names, motion_ids = _build_vote_matrix(
+        motion_db, "2020-01-01", "2020-01-10"
+    )
+    assert mat.shape == (5, 6)
+
+    # _safe_k: with k=10 -> min_dim=5 -> returns 4
+    assert _safe_k(mat, 10) == 4
+    assert _safe_k(mat, 3) == 3
+
+    # run_svd_for_window with k=10 -> should use k_used=4
+    res = run_svd_for_window(motion_db, "w1", "2020-01-01", "2020-01-10", k=10)
+    assert res["k_used"] == 4
+    assert res["stored_mp"] == 5
+    assert res["stored_motion"] == 6
+
+
+def test_procrustes_align():
+    np.random.seed(0)
+    # create reference anchors and current anchors rotated + noise
+    ref = np.random.randn(10, 3)
+    # create orthogonal rotation
+    Q, _ = np.linalg.qr(np.random.randn(3, 3))
+    cur = ref.dot(Q) + 0.1 * np.random.randn(10, 3)
+
+    before = np.linalg.norm(cur - ref)
+    transformed = _procrustes_align(ref, cur)
+    after = np.linalg.norm(transformed - ref)
+
+    assert after < before

tests/test_text_pipeline.py

@@ -0,0 +1,80 @@
+import json
+import pytest
+
+# duckdb is an optional dependency in some environments; skip test if not available
+duckdb = pytest.importorskip("duckdb")
+
+from database import MotionDatabase
+
+
+def test_ensure_text_embeddings_monkeypatch(tmp_path, monkeypatch):
+    # prepare temp db
+    db_path = str(tmp_path / "motions.db")
+    db = MotionDatabase(db_path)
+
+    # create embeddings table (migration would normally do this)
+    conn = duckdb.connect(db.db_path)
+    # create embeddings table with autoincrement id for sqlite
+    conn.execute("CREATE SEQUENCE IF NOT EXISTS embeddings_id_seq START 1")
+    conn.execute(
+        "CREATE TABLE IF NOT EXISTS embeddings (id INTEGER PRIMARY KEY DEFAULT nextval('embeddings_id_seq'), motion_id INTEGER, model TEXT, vector JSON, created_at TIMESTAMP)"
+    )
+
+    # insert three motions
+    conn.execute(
+        "INSERT INTO motions (title, description, url, layman_explanation) VALUES (?, ?, ?, ?)",
+        ("t1", "d1", "u1", "ex1"),
+    )
+    conn.execute(
+        "INSERT INTO motions (title, description, url, layman_explanation) VALUES (?, ?, ?, ?)",
+        ("t2", "d2", "u2", "ex2"),
+    )
+    conn.execute(
+        "INSERT INTO motions (title, description, url, layman_explanation) VALUES (?, ?, ?, ?)",
+        ("t3", "d3", "u3", "ex3"),
+    )
+
+    # fetch ids
+    rows = conn.execute("SELECT id FROM motions ORDER BY id").fetchall()
+    ids = [r[0] for r in rows]
+
+    # insert existing embedding for first motion
+    import json as _json
+
+    vec = _json.dumps([0.1] * 16)
+    conn.execute(
+        "INSERT INTO embeddings (motion_id, model, vector) VALUES (?, ?, ?)",
+        (ids[0], "test-model", vec),
+    )
+
+    conn.close()
+
+    # monkeypatch ai_provider.get_embedding
+    def fake_get_embedding(text, model=None):
+        return [0.1] * 16
+
+    monkeypatch.setattr("ai_provider.get_embedding", fake_get_embedding)
+
+    # run ensure_text_embeddings
+    from pipeline.text_pipeline import ensure_text_embeddings
+
+    stored, skipped_existing, skipped_no_text, errors = ensure_text_embeddings(
+        db_path=db_path, model="test-model"
+    )
+
+    assert stored == 2
+    assert skipped_existing == 1
+    assert skipped_no_text == 0
+    assert errors == 0
+
+    # verify stored vectors length
+    conn = duckdb.connect(db.db_path)
+    rows = conn.execute(
+        "SELECT vector FROM embeddings WHERE model = ? ORDER BY motion_id",
+        ("test-model",),
+    ).fetchall()
+    conn.close()
+    assert len(rows) == 3
+    for r in rows:
+        v = _json.loads(r[0])
+        assert len(v) == 16

tests/types/test_motion_types.py

@@ -0,0 +1,22 @@
+import json
+
+from src.types.motion_types import SimilarityNeighbor, to_json, from_json
+
+
+def test_similarity_neighbor_json_roundtrip():
+    neighbors = [
+        SimilarityNeighbor(motion_id="m1", score=0.9),
+        SimilarityNeighbor(motion_id="m2", score=0.75),
+    ]
+
+    # Serialize to JSON string
+    json_str = to_json(neighbors)
+    assert isinstance(json_str, str)
+
+    # Ensure it's valid JSON
+    parsed = json.loads(json_str)
+    assert isinstance(parsed, list)
+
+    # Deserialize back to objects
+    recovered = from_json(json_str)
+    assert recovered == neighbors

tests/utils/migration_fixtures.py

@@ -0,0 +1,66 @@
+"""
+Test helper fixtures for database migrations.
+
+Provides a pytest fixture `test_db` that inspects the environment variable
+`TEST_DB_URL` to decide what to yield:
+
+- If `TEST_DB_URL` is not set, the fixture yields None. This allows tests to
+  be skipped or operate in a no-database mode in CI or local runs where a
+  test database is not available.
+- If `TEST_DB_URL` is set and starts with "sqlite", an sqlite3 connection is
+  created via `sqlite3.connect` and yielded. The connection is closed after
+  the test completes.
+
+Decision: keep this fixture lightweight and focused on sqlite for local
+smoke-testing. If other database backends are needed later, expand this
+fixture accordingly.
+"""
+
+from typing import Optional
+import os
+import sqlite3
+
+import pytest
+
+
+@pytest.fixture
+def test_db():
+    """Yield a test database connection or None.
+
+    Behavior:
+    - If TEST_DB_URL is not set in the environment, yield None.
+    - If TEST_DB_URL is set and begins with 'sqlite', open an sqlite3
+      connection and yield it. The connection will be closed when the test
+      finishes.
+    """
+    url = os.environ.get("TEST_DB_URL")
+    if not url:
+        yield None
+        return
+
+    # Only support sqlite URLs in this lightweight fixture.
+    if url.startswith("sqlite"):
+        # For sqlite URLs, accept either a bare file path or a file:// style
+        # URL. sqlite3.connect handles file paths; if a file:// prefix is
+        # present, strip it.
+        path = url
+        if path.startswith("sqlite:///"):
+            # sqlite:///path => /path
+            path = path[len("sqlite:///") :]
+        elif path.startswith("sqlite://"):
+            path = path[len("sqlite://") :]
+
+        conn = sqlite3.connect(path)
+        try:
+            yield conn
+        finally:
+            try:
+                conn.close()
+            except Exception:
+                # Best-effort close; tests shouldn't fail on close errors.
+                pass
+        return
+
+    # Unknown or unsupported TEST_DB_URL scheme — yield None to keep tests
+    # tolerant in environments where the fixture can't create a connection.
+    yield None