feat(mp-quiz): add MP quiz tab and DB helpers; add design and plan docs

pipeline/text_pipeline.py

@@ -63,23 +63,28 @@ def _select_text(
 def ensure_text_embeddings(
     db_path: Optional[str] = None,
     model: Optional[str] = None,
-    batch_size: int = 50,
+    batch_size: int = 128,
     db=None,
     embedder=None,
+    min_batch_size: int = 4,
+    max_batch_size: int = 512,
+    growth_factor: float = 1.5,
 ) -> Tuple[int, int, int, int, list]:
     """Ensure all motions have text embeddings for `model`.
 
-    Uses batched API calls (batch_size texts per HTTP request) for speed.
-    Returns tuple (stored_count, skipped_existing, skipped_no_text, errors).
+    Uses AIMD batch sizing to maximise throughput:
+      - After each fully-successful batch: grow by growth_factor (probe upward).
+      - After any batch failure: halve (back off from API limits).
+    This converges on the largest batch the provider can reliably handle.
+
+    Returns tuple (stored_count, skipped_existing, skipped_no_text, errors, failed_ids).
     """
     model = model or DEFAULT_MODEL
     if db is None:
         db = MotionDatabase(db_path) if db_path else default_db
 
-    # motions to process
     to_process = _select_text(db, model)
 
-    # how many already exist
     if duckdb is None:
         total_motions = 0
         existing = 0
@@ -89,7 +94,6 @@ def ensure_text_embeddings(
             total_motions = conn.execute("SELECT COUNT(*) FROM motions").fetchone()[0]
         except Exception:
             total_motions = 0
-
         try:
             existing = conn.execute(
                 "SELECT COUNT(DISTINCT motion_id) FROM embeddings WHERE model = ?",
@@ -97,7 +101,6 @@ def ensure_text_embeddings(
             ).fetchone()[0]
         except Exception:
             existing = 0
-
         conn.close()
 
     stored = 0
@@ -105,7 +108,6 @@ def ensure_text_embeddings(
     errors = 0
     failed_ids: list = []
 
-    # Separate motions with text from those without
     with_text: List[Tuple[int, str]] = []
     for motion_id, text in to_process:
         if not text:
@@ -114,17 +116,23 @@ def ensure_text_embeddings(
         else:
             with_text.append((motion_id, text))
 
+    current_batch_size = max(min_batch_size, min(max_batch_size, batch_size))
     _logger.info(
-        "Processing %d motions in batches of %d (%d skipped no text, %d already exist)",
+        "Processing %d motions (initial_batch=%d, min=%d, max=%d, growth=%.1fx"
+        " — %d skipped no text, %d already exist)",
         len(with_text),
-        batch_size,
+        current_batch_size,
+        min_batch_size,
+        max_batch_size,
+        growth_factor,
         skipped_no_text,
         existing,
     )
 
-    # Process in batches
-    for batch_start in range(0, len(with_text), batch_size):
-        batch = with_text[batch_start : batch_start + batch_size]
+    i = 0
+    n = len(with_text)
+    while i < n:
+        batch = with_text[i : i + current_batch_size]
         batch_ids = [mid for mid, _ in batch]
         batch_texts = [txt for _, txt in batch]
 
@@ -132,20 +140,21 @@ def ensure_text_embeddings(
             batch_texts,
             motion_ids=batch_ids,
             model=model,
-            batch_size=batch_size,
+            batch_size=current_batch_size,
             embedder=embedder,
         )
 
         batch_stored = 0
+        batch_errors = 0
         for (motion_id, _text), vec in zip(batch, vecs):
             if not isinstance(vec, list):
                 _logger.warning(
                     "Embedding provider returned non-list for motion %s", motion_id
                 )
                 errors += 1
+                batch_errors += 1
                 failed_ids.append(motion_id)
                 continue
-
             try:
                 res = db.store_embedding(motion_id, model, vec)
                 if res and res > 0:
@@ -158,24 +167,40 @@ def ensure_text_embeddings(
                         res,
                     )
                     errors += 1
+                    batch_errors += 1
                     failed_ids.append(motion_id)
             except Exception as exc:
                 _logger.error(
                     "Error storing embedding for motion %s: %s", motion_id, exc
                 )
                 errors += 1
+                batch_errors += 1
                 failed_ids.append(motion_id)
 
+        # AIMD: grow on full success, halve on any failure
+        prev_batch_size = current_batch_size
+        if batch_errors == 0:
+            current_batch_size = min(
+                max_batch_size, int(current_batch_size * growth_factor)
+            )
+        else:
+            current_batch_size = max(min_batch_size, current_batch_size // 2)
+
         _logger.info(
-            "Batch %d-%d: stored %d/%d (total: %d/%d)",
-            batch_start,
-            batch_start + len(batch),
+            "Batch %d-%d: stored %d/%d errors %d — batch_size %d→%d (total: %d/%d)",
+            i,
+            i + len(batch),
             batch_stored,
             len(batch),
+            batch_errors,
+            prev_batch_size,
+            current_batch_size,
             stored + existing,
             total_motions,
         )
 
+        i += len(batch)
+
     skipped_existing = int(existing)
     return stored, skipped_existing, skipped_no_text, errors, failed_ids