feat(pipeline): implement parliamentary embedding pipeline MVP
- Add 4 migration files: mp_votes, mp_metadata, svd_vectors, fused_embeddings - Extend database.py with 5 new helper methods and table init - Add pipeline/ package: extract_mp_votes, fetch_mp_metadata, text_pipeline, svd_pipeline (with Procrustes alignment), fusion - Add full test suite (17 tests) covering all pipeline modules and migrations - Fix Procrustes alignment bug: scipy scale is a norm value, not a multiplier - Fix DuckDB date type handling in test assertions (datetime.date vs string) - Remove duckdb.py shim; tests now run against real duckdb + scipy via uv Ref: thoughts/shared/plans/2026-03-21-parliamentary-embedding-pipeline-plan.md
This commit is contained in:
@@ -0,0 +1,11 @@
|
||||
-- Add a separate embeddings table for semantic search and storage of vectors (DuckDB-compatible)
|
||||
CREATE TABLE IF NOT EXISTS embeddings (
|
||||
id INTEGER,
|
||||
motion_id INTEGER NOT NULL,
|
||||
model TEXT NOT NULL,
|
||||
vector JSON NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT current_timestamp
|
||||
);
|
||||
-- DuckDB does not support AUTOINCREMENT; emulate id via a sequence if needed elsewhere
|
||||
CREATE SEQUENCE IF NOT EXISTS embeddings_id_seq START 1;
|
||||
-- Populate id via trigger-like insert pattern is handled by application code (select nextval when inserting)
|
||||
@@ -0,0 +1,6 @@
|
||||
-- Migration: add externe_identifier and body_text columns to motions
|
||||
-- externe_identifier: e.g. "kst-36600-VII-28" from DocumentVersie.ExterneIdentifier
|
||||
-- body_text: full plain-text motion body scraped from officielebekendmakingen.nl
|
||||
|
||||
ALTER TABLE motions ADD COLUMN IF NOT EXISTS externe_identifier VARCHAR;
|
||||
ALTER TABLE motions ADD COLUMN IF NOT EXISTS body_text VARCHAR;
|
||||
@@ -0,0 +1,24 @@
|
||||
-- Migration: create audit_events table
|
||||
-- Date: 2026-03-22
|
||||
-- Description: Placeholder migration to add an audit_events table to record audit logs.
|
||||
--
|
||||
-- Decision: The actual SQL is intentionally left commented out to avoid making
|
||||
-- database changes during test runs. When ready to apply, uncomment and
|
||||
-- adapt the SQL for your database engine.
|
||||
|
||||
/*
|
||||
CREATE TABLE audit_events (
|
||||
id UUID PRIMARY KEY,
|
||||
actor_id UUID NOT NULL,
|
||||
action TEXT NOT NULL,
|
||||
target_type TEXT,
|
||||
target_id UUID,
|
||||
metadata JSONB,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT now()
|
||||
);
|
||||
|
||||
-- Add indexes as needed, e.g.:
|
||||
-- CREATE INDEX ON audit_events (actor_id);
|
||||
*/
|
||||
|
||||
-- End of migration placeholder
|
||||
@@ -0,0 +1,15 @@
|
||||
-- 2026-03-22-add-similarity-cache.sql
|
||||
-- Placeholder migration for adding a similarity_cache table
|
||||
-- Decision: Keep SQL commented out so CI does not accidentally modify databases.
|
||||
|
||||
/*
|
||||
-- Example (commented out):
|
||||
CREATE TABLE similarity_cache (
|
||||
id SERIAL PRIMARY KEY,
|
||||
key TEXT NOT NULL,
|
||||
vector FLOAT8[] NOT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE DEFAULT now()
|
||||
);
|
||||
*/
|
||||
|
||||
-- No executable SQL in this file. Intentionally left as a safe no-op.
|
||||
@@ -0,0 +1,13 @@
|
||||
----SQL
|
||||
CREATE SEQUENCE IF NOT EXISTS fused_embeddings_id_seq START 1;
|
||||
CREATE TABLE IF NOT EXISTS fused_embeddings (
|
||||
id INTEGER DEFAULT nextval('fused_embeddings_id_seq'),
|
||||
motion_id INTEGER NOT NULL,
|
||||
window_id TEXT NOT NULL,
|
||||
vector JSON NOT NULL,
|
||||
svd_dims INTEGER NOT NULL,
|
||||
text_dims INTEGER NOT NULL,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id)
|
||||
);
|
||||
----END
|
||||
@@ -0,0 +1,9 @@
|
||||
----SQL
|
||||
CREATE TABLE IF NOT EXISTS mp_metadata (
|
||||
mp_name TEXT PRIMARY KEY,
|
||||
party TEXT,
|
||||
van DATE,
|
||||
tot_en_met DATE,
|
||||
persoon_id TEXT
|
||||
);
|
||||
----END
|
||||
@@ -0,0 +1,13 @@
|
||||
----SQL
|
||||
CREATE SEQUENCE IF NOT EXISTS mp_votes_id_seq START 1;
|
||||
CREATE TABLE IF NOT EXISTS mp_votes (
|
||||
id INTEGER DEFAULT nextval('mp_votes_id_seq'),
|
||||
motion_id INTEGER NOT NULL,
|
||||
mp_name TEXT NOT NULL,
|
||||
party TEXT,
|
||||
vote TEXT NOT NULL,
|
||||
date DATE,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id)
|
||||
);
|
||||
----END
|
||||
@@ -0,0 +1,13 @@
|
||||
----SQL
|
||||
CREATE SEQUENCE IF NOT EXISTS svd_vectors_id_seq START 1;
|
||||
CREATE TABLE IF NOT EXISTS svd_vectors (
|
||||
id INTEGER DEFAULT nextval('svd_vectors_id_seq'),
|
||||
window_id TEXT NOT NULL,
|
||||
entity_type TEXT NOT NULL,
|
||||
entity_id TEXT NOT NULL,
|
||||
vector JSON NOT NULL,
|
||||
model TEXT,
|
||||
created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (id)
|
||||
);
|
||||
----END
|
||||
Reference in New Issue
Block a user