feat(similarity): add precomputed similarity cache, fix fusion N+1, add 429 retry

- Add similarity/ package (compute.py, lookup.py) with numpy-based
  pairwise cosine similarity and cached lookup
- database.py: create embeddings + similarity_cache tables in _init_database(),
  add store_similarity_batch/get_cached_similarities/clear_similarity_cache helpers
- pipeline/fusion.py: replace N+1 per-motion embedding SELECT with single
  bulk JOIN using DuckDB QUALIFY window function
- ai_provider.py: retry HTTP 429 with Retry-After header support
- migrations/2026-03-22-add-similarity-cache.sql: make executable
- Add tests for similarity compute, db helpers, and 429 retry (34 pass, 2 skip)
This commit is contained in:
2026-03-22 03:02:25 +01:00
parent a248807e03
commit a78bee9b0a
11 changed files with 879 additions and 38 deletions
+16 -12
View File
@@ -1,15 +1,19 @@
-- 2026-03-22-add-similarity-cache.sql
-- Placeholder migration for adding a similarity_cache table
-- Decision: Keep SQL commented out so CI does not accidentally modify databases.
-- 2026-03-22-add-similarity-cache.sql - similarity migration
-- This migration creates a sequence and the similarity_cache table.
/*
-- Example (commented out):
CREATE TABLE similarity_cache (
id SERIAL PRIMARY KEY,
key TEXT NOT NULL,
vector FLOAT8[] NOT NULL,
created_at TIMESTAMP WITH TIME ZONE DEFAULT now()
-- Create a sequence for generating integer ids (DuckDB compatible)
CREATE SEQUENCE IF NOT EXISTS similarity_cache_id_seq START 1;
-- Create the similarity_cache table.
CREATE TABLE IF NOT EXISTS similarity_cache (
id INTEGER DEFAULT nextval('similarity_cache_id_seq'),
source_motion_id INTEGER NOT NULL,
target_motion_id INTEGER NOT NULL,
score REAL NOT NULL,
vector_type TEXT NOT NULL,
window_id TEXT,
created_at TIMESTAMP DEFAULT current_timestamp
);
*/
-- No executable SQL in this file. Intentionally left as a safe no-op.
-- Optionally create an index to speed lookups (no-op if already exists)
CREATE INDEX IF NOT EXISTS idx_similarity_cache_source_target ON similarity_cache (source_motion_id, target_motion_id);