feat(pipeline): implement parliamentary embedding pipeline MVP
- Add 4 migration files: mp_votes, mp_metadata, svd_vectors, fused_embeddings - Extend database.py with 5 new helper methods and table init - Add pipeline/ package: extract_mp_votes, fetch_mp_metadata, text_pipeline, svd_pipeline (with Procrustes alignment), fusion - Add full test suite (17 tests) covering all pipeline modules and migrations - Fix Procrustes alignment bug: scipy scale is a norm value, not a multiplier - Fix DuckDB date type handling in test assertions (datetime.date vs string) - Remove duckdb.py shim; tests now run against real duckdb + scipy via uv Ref: thoughts/shared/plans/2026-03-21-parliamentary-embedding-pipeline-plan.md
This commit is contained in:
@@ -0,0 +1,87 @@
|
||||
import json
|
||||
import os
|
||||
import numpy as np
|
||||
import pytest
|
||||
|
||||
# duckdb is an optional dependency in some environments; skip test if not available
|
||||
duckdb = pytest.importorskip("duckdb")
|
||||
|
||||
|
||||
def test_pipeline_end_to_end(tmp_path, monkeypatch):
|
||||
# ensure determinism for any random embedding generation
|
||||
np.random.seed(0)
|
||||
|
||||
# prepare temp db
|
||||
db_path = str(tmp_path / "motions.db")
|
||||
|
||||
# create the minimal MotionDatabase schema using existing code where possible
|
||||
from database import MotionDatabase
|
||||
|
||||
db = MotionDatabase(db_path)
|
||||
|
||||
# create embeddings table (migration would normally do this)
|
||||
conn = duckdb.connect(db.db_path)
|
||||
conn.execute("CREATE SEQUENCE IF NOT EXISTS embeddings_id_seq START 1")
|
||||
conn.execute(
|
||||
"CREATE TABLE IF NOT EXISTS embeddings (id INTEGER PRIMARY KEY DEFAULT nextval('embeddings_id_seq'), motion_id INTEGER, model TEXT, vector JSON, created_at TIMESTAMP)"
|
||||
)
|
||||
|
||||
# insert three motions
|
||||
conn.execute(
|
||||
"INSERT INTO motions (title, description, url, layman_explanation) VALUES (?, ?, ?, ?)",
|
||||
("t1", "d1", "u1", "ex1"),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO motions (title, description, url, layman_explanation) VALUES (?, ?, ?, ?)",
|
||||
("t2", "d2", "u2", "ex2"),
|
||||
)
|
||||
conn.execute(
|
||||
"INSERT INTO motions (title, description, url, layman_explanation) VALUES (?, ?, ?, ?)",
|
||||
("t3", "d3", "u3", "ex3"),
|
||||
)
|
||||
|
||||
# fetch ids
|
||||
rows = conn.execute("SELECT id FROM motions ORDER BY id").fetchall()
|
||||
ids = [r[0] for r in rows]
|
||||
|
||||
# insert existing embedding for first motion
|
||||
vec = json.dumps([0.1] * 16)
|
||||
conn.execute(
|
||||
"INSERT INTO embeddings (motion_id, model, vector) VALUES (?, ?, ?)",
|
||||
(ids[0], "test-model", vec),
|
||||
)
|
||||
|
||||
conn.close()
|
||||
|
||||
# monkeypatch ai_provider.get_embedding to deterministic vector
|
||||
import ai_provider
|
||||
|
||||
def fake_get_embedding(text, model=None):
|
||||
# produce a deterministic vector based on seeded numpy
|
||||
return list(np.random.rand(16))
|
||||
|
||||
monkeypatch.setattr("ai_provider.get_embedding", fake_get_embedding)
|
||||
|
||||
# run ensure_text_embeddings
|
||||
from pipeline.text_pipeline import ensure_text_embeddings
|
||||
|
||||
stored, skipped_existing, skipped_no_text, errors = ensure_text_embeddings(
|
||||
db_path=db_path, model="test-model"
|
||||
)
|
||||
|
||||
assert stored == 2
|
||||
assert skipped_existing == 1
|
||||
assert skipped_no_text == 0
|
||||
assert errors == 0
|
||||
|
||||
# verify stored vectors length
|
||||
conn = duckdb.connect(db.db_path)
|
||||
rows = conn.execute(
|
||||
"SELECT vector FROM embeddings WHERE model = ? ORDER BY motion_id",
|
||||
("test-model",),
|
||||
).fetchall()
|
||||
conn.close()
|
||||
assert len(rows) == 3
|
||||
for r in rows:
|
||||
v = json.loads(r[0])
|
||||
assert len(v) == 16
|
||||
Reference in New Issue
Block a user