feat(pipeline): implement parliamentary embedding pipeline MVP

- Add 4 migration files: mp_votes, mp_metadata, svd_vectors, fused_embeddings
- Extend database.py with 5 new helper methods and table init
- Add pipeline/ package: extract_mp_votes, fetch_mp_metadata, text_pipeline,
  svd_pipeline (with Procrustes alignment), fusion
- Add full test suite (17 tests) covering all pipeline modules and migrations
- Fix Procrustes alignment bug: scipy scale is a norm value, not a multiplier
- Fix DuckDB date type handling in test assertions (datetime.date vs string)
- Remove duckdb.py shim; tests now run against real duckdb + scipy via uv

Ref: thoughts/shared/plans/2026-03-21-parliamentary-embedding-pipeline-plan.md
This commit is contained in:
2026-03-21 22:31:22 +01:00
parent c498c3467e
commit a36e6cba4e
68 changed files with 6822 additions and 0 deletions
+11
View File
@@ -0,0 +1,11 @@
-- Add a separate embeddings table for semantic search and storage of vectors (DuckDB-compatible)
CREATE TABLE IF NOT EXISTS embeddings (
id INTEGER,
motion_id INTEGER NOT NULL,
model TEXT NOT NULL,
vector JSON NOT NULL,
created_at TIMESTAMP DEFAULT current_timestamp
);
-- DuckDB does not support AUTOINCREMENT; emulate id via a sequence if needed elsewhere
CREATE SEQUENCE IF NOT EXISTS embeddings_id_seq START 1;
-- Populate id via trigger-like insert pattern is handled by application code (select nextval when inserting)
+6
View File
@@ -0,0 +1,6 @@
-- Migration: add externe_identifier and body_text columns to motions
-- externe_identifier: e.g. "kst-36600-VII-28" from DocumentVersie.ExterneIdentifier
-- body_text: full plain-text motion body scraped from officielebekendmakingen.nl
ALTER TABLE motions ADD COLUMN IF NOT EXISTS externe_identifier VARCHAR;
ALTER TABLE motions ADD COLUMN IF NOT EXISTS body_text VARCHAR;
@@ -0,0 +1,24 @@
-- Migration: create audit_events table
-- Date: 2026-03-22
-- Description: Placeholder migration to add an audit_events table to record audit logs.
--
-- Decision: The actual SQL is intentionally left commented out to avoid making
-- database changes during test runs. When ready to apply, uncomment and
-- adapt the SQL for your database engine.
/*
CREATE TABLE audit_events (
id UUID PRIMARY KEY,
actor_id UUID NOT NULL,
action TEXT NOT NULL,
target_type TEXT,
target_id UUID,
metadata JSONB,
created_at TIMESTAMP WITH TIME ZONE DEFAULT now()
);
-- Add indexes as needed, e.g.:
-- CREATE INDEX ON audit_events (actor_id);
*/
-- End of migration placeholder
@@ -0,0 +1,15 @@
-- 2026-03-22-add-similarity-cache.sql
-- Placeholder migration for adding a similarity_cache table
-- Decision: Keep SQL commented out so CI does not accidentally modify databases.
/*
-- Example (commented out):
CREATE TABLE similarity_cache (
id SERIAL PRIMARY KEY,
key TEXT NOT NULL,
vector FLOAT8[] NOT NULL,
created_at TIMESTAMP WITH TIME ZONE DEFAULT now()
);
*/
-- No executable SQL in this file. Intentionally left as a safe no-op.
@@ -0,0 +1,13 @@
----SQL
CREATE SEQUENCE IF NOT EXISTS fused_embeddings_id_seq START 1;
CREATE TABLE IF NOT EXISTS fused_embeddings (
id INTEGER DEFAULT nextval('fused_embeddings_id_seq'),
motion_id INTEGER NOT NULL,
window_id TEXT NOT NULL,
vector JSON NOT NULL,
svd_dims INTEGER NOT NULL,
text_dims INTEGER NOT NULL,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id)
);
----END
@@ -0,0 +1,9 @@
----SQL
CREATE TABLE IF NOT EXISTS mp_metadata (
mp_name TEXT PRIMARY KEY,
party TEXT,
van DATE,
tot_en_met DATE,
persoon_id TEXT
);
----END
@@ -0,0 +1,13 @@
----SQL
CREATE SEQUENCE IF NOT EXISTS mp_votes_id_seq START 1;
CREATE TABLE IF NOT EXISTS mp_votes (
id INTEGER DEFAULT nextval('mp_votes_id_seq'),
motion_id INTEGER NOT NULL,
mp_name TEXT NOT NULL,
party TEXT,
vote TEXT NOT NULL,
date DATE,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id)
);
----END
@@ -0,0 +1,13 @@
----SQL
CREATE SEQUENCE IF NOT EXISTS svd_vectors_id_seq START 1;
CREATE TABLE IF NOT EXISTS svd_vectors (
id INTEGER DEFAULT nextval('svd_vectors_id_seq'),
window_id TEXT NOT NULL,
entity_type TEXT NOT NULL,
entity_id TEXT NOT NULL,
vector JSON NOT NULL,
model TEXT,
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (id)
);
----END