cleanup: remove stale .mindmodel, old venvs, orphaned code, and transient artifacts
Removes: - .mindmodel/ directory and related CI workflows (mindmodel-schedule.yml, mindmodel-validation.yml) - scripts/mindmodel/ and scripts/validate_mindmodel.py - src/types/ and src/validators/ (orphaned type modules, only used by mindmodel) - tests/ci/, tests/scripts/mindmodel/, tests/types/, tests/validators/ (mindmodel-only tests) - thoughts/ledgers/ and thoughts/shared/ (stale transient directories) - .venv_axis and .venv_plotly (orphaned virtual environments, ~1.1 GB) - outputs/blog-charts/ (stale generated HTML files) - data/*.json sidecars (empty cache artifacts) - __pycache__ and *.pyc files across repo Updates: - .gitignore: remove thoughts/shared/analyses/ entry Space reclaimed: ~1.1 GB+
This commit is contained in:
@@ -1,72 +0,0 @@
|
||||
import os
|
||||
import re
|
||||
from typing import List
|
||||
|
||||
|
||||
def file_exists(base_dir: str, path: str) -> bool:
|
||||
"""Check whether a path exists under base_dir without opening the file.
|
||||
|
||||
This resolves the path relative to base_dir and returns True if the
|
||||
resolved path exists on the filesystem (file or directory).
|
||||
"""
|
||||
if not base_dir:
|
||||
base = ""
|
||||
else:
|
||||
base = base_dir
|
||||
full = os.path.join(base, path)
|
||||
return os.path.exists(full)
|
||||
|
||||
|
||||
def detect_truncated(snippet: str) -> bool:
|
||||
"""Heuristic detection whether a snippet is truncated.
|
||||
|
||||
Returns True if the snippet ends with an ellipsis '...' (after
|
||||
trimming whitespace) or contains a common truncation marker like
|
||||
the substring 'truncat' (case-insensitive).
|
||||
"""
|
||||
if snippet is None:
|
||||
return False
|
||||
s = snippet.strip()
|
||||
if s.endswith("..."):
|
||||
return True
|
||||
if "truncat" in s.lower():
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
def find_potential_secrets(text: str) -> List[str]:
|
||||
"""Scan the provided text and return a list of potential secret-like
|
||||
strings. This uses a few common heuristics and regex patterns and only
|
||||
scans the provided text (no external resources).
|
||||
|
||||
The function returns a list of found token strings (values when
|
||||
capture groups are available, otherwise the matched substring).
|
||||
"""
|
||||
if not text:
|
||||
return []
|
||||
|
||||
candidates: List[str] = []
|
||||
|
||||
# AWS access key id pattern (common): AKIA followed by 16 alphanumeric
|
||||
aws_pattern = re.compile(r"AKIA[0-9A-Z]{16}")
|
||||
candidates.extend(aws_pattern.findall(text))
|
||||
|
||||
# Common key/value patterns like api_key = "..." or "api-key: ..."
|
||||
# allow shorter secret values (down to 4 chars) to catch short test values
|
||||
kv_pattern = re.compile(
|
||||
r"(?i)(?:api[_-]?key|secret[_-]?key|access[_-]?token|access[_-]?key|token|password|passwd|pwd)\s*[=:]+\s*['\"]?([A-Za-z0-9\-_=+/\.]{4,128})['\"]?"
|
||||
)
|
||||
candidates.extend(m.group(1) for m in kv_pattern.finditer(text))
|
||||
|
||||
# Generic long hex or base64-like strings (heuristic)
|
||||
long_hex = re.compile(r"\b([a-f0-9]{32,128})\b", re.IGNORECASE)
|
||||
candidates.extend(long_hex.findall(text))
|
||||
|
||||
# Deduplicate while preserving order
|
||||
seen = set()
|
||||
result: List[str] = []
|
||||
for c in candidates:
|
||||
if c and c not in seen:
|
||||
seen.add(c)
|
||||
result.append(c)
|
||||
return result
|
||||
@@ -1,32 +0,0 @@
|
||||
from typing import List, Optional
|
||||
|
||||
|
||||
def main(argv: Optional[List[str]] = None) -> int:
|
||||
"""CLI wrapper that delegates to scripts.mindmodel.validator.main.
|
||||
|
||||
Returns the integer exit code from the delegated main. If the
|
||||
validator module is not available or raises, return a non-zero
|
||||
exit code.
|
||||
"""
|
||||
try:
|
||||
# Import here to avoid side-effects on module import
|
||||
from scripts.mindmodel import validator
|
||||
|
||||
# Call the validator.main if present
|
||||
if hasattr(validator, "main"):
|
||||
result = validator.main(argv)
|
||||
# Ensure we return an int
|
||||
try:
|
||||
return int(result) # type: ignore
|
||||
except Exception:
|
||||
return 1
|
||||
else:
|
||||
return 2
|
||||
except Exception:
|
||||
# Import error or runtime error — return non-zero so callers
|
||||
# can detect failure (tests expect non-zero on missing manifest)
|
||||
return 2
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
@@ -1,67 +0,0 @@
|
||||
"""Simple manifest loader for mindmodel manifests.
|
||||
|
||||
Provides `load_manifest(path: str) -> dict` and `ManifestLoadError`.
|
||||
|
||||
Behavior:
|
||||
- If PyYAML is installed, uses yaml.safe_load to parse the file.
|
||||
- Otherwise falls back to the stdlib json parser.
|
||||
- If the top-level document is a list it will be normalized to {"constraints": <list>}.
|
||||
- Raises ManifestLoadError for missing file or parse errors.
|
||||
"""
|
||||
|
||||
from typing import Any, Dict
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ManifestLoadError(Exception):
|
||||
"""Raised when a manifest cannot be loaded or parsed."""
|
||||
|
||||
|
||||
try:
|
||||
import yaml # type: ignore
|
||||
except Exception: # YAML not available
|
||||
yaml = None # type: ignore
|
||||
|
||||
|
||||
def _parse_with_yaml(text: str) -> Any:
|
||||
# yamlsafe_load may return any Python structure
|
||||
try:
|
||||
return yaml.safe_load(text)
|
||||
except Exception as exc: # pragma: no cover - defensive
|
||||
raise ManifestLoadError(f"YAML parse error: {exc}") from exc
|
||||
|
||||
|
||||
def _parse_with_json(text: str) -> Any:
|
||||
try:
|
||||
return json.loads(text)
|
||||
except Exception as exc:
|
||||
raise ManifestLoadError(f"JSON parse error: {exc}") from exc
|
||||
|
||||
|
||||
def load_manifest(path: str) -> Dict[str, Any]:
|
||||
"""Load a manifest from the given file path and normalize it to a dict.
|
||||
|
||||
If the top-level document is a list, it will be returned as {"constraints": list}.
|
||||
Raises ManifestLoadError if the file does not exist or if parsing fails.
|
||||
"""
|
||||
p = Path(path)
|
||||
if not p.exists():
|
||||
raise ManifestLoadError(f"Manifest file not found: {path}")
|
||||
|
||||
text = p.read_text(encoding="utf-8")
|
||||
|
||||
if yaml is not None:
|
||||
data = _parse_with_yaml(text)
|
||||
else:
|
||||
data = _parse_with_json(text)
|
||||
|
||||
# Normalize
|
||||
if isinstance(data, list):
|
||||
return {"constraints": data}
|
||||
|
||||
if isinstance(data, dict):
|
||||
return data
|
||||
|
||||
# Unexpected top-level type, wrap it
|
||||
return {"manifest": data}
|
||||
@@ -1,108 +0,0 @@
|
||||
from typing import Dict, Tuple, List, Any
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.mindmodel import loader
|
||||
from scripts.mindmodel import checks
|
||||
|
||||
|
||||
def validate_manifest(path: str, base_dir: str = None) -> Tuple[int, Dict[str, Any]]:
|
||||
"""Validate a manifest file at `path`.
|
||||
|
||||
Returns a tuple (exit_code, report).
|
||||
|
||||
exit codes:
|
||||
0 - ok (no issues)
|
||||
1 - warnings (only truncated snippets found)
|
||||
2 - critical (missing files, secrets, or parse error)
|
||||
"""
|
||||
report: Dict[str, Any] = {
|
||||
"path": path,
|
||||
"secrets": [],
|
||||
"missing_files": [],
|
||||
"truncated": 0,
|
||||
"constraints": [],
|
||||
}
|
||||
|
||||
p = Path(path)
|
||||
try:
|
||||
raw_text = p.read_text(encoding="utf-8")
|
||||
except Exception as exc:
|
||||
report["load_error"] = f"Manifest file not readable: {exc}"
|
||||
return 2, report
|
||||
|
||||
# scan for secrets in the manifest text
|
||||
secrets = checks.find_potential_secrets(raw_text)
|
||||
report["secrets"] = secrets
|
||||
|
||||
try:
|
||||
manifest = loader.load_manifest(path)
|
||||
except loader.ManifestLoadError as exc:
|
||||
report["load_error"] = str(exc)
|
||||
# treat parse/load errors as critical
|
||||
return 2, report
|
||||
|
||||
constraints = manifest.get("constraints") or []
|
||||
|
||||
for constraint in constraints:
|
||||
c_rep: Dict[str, Any] = {"constraint": constraint, "evidence": []}
|
||||
for ev in (
|
||||
constraint.get("evidence", [])
|
||||
if isinstance(constraint.get("evidence", []), list)
|
||||
else []
|
||||
):
|
||||
text = ev.get("text") if isinstance(ev, dict) else None
|
||||
file_ref = ev.get("file") if isinstance(ev, dict) else None
|
||||
|
||||
exists = True
|
||||
if file_ref:
|
||||
if not checks.file_exists(base_dir or "", file_ref):
|
||||
exists = False
|
||||
report["missing_files"].append(file_ref)
|
||||
|
||||
truncated = False
|
||||
if text:
|
||||
truncated = checks.detect_truncated(text)
|
||||
if truncated:
|
||||
report["truncated"] += 1
|
||||
|
||||
c_rep["evidence"].append(
|
||||
{
|
||||
"text": text,
|
||||
"file": file_ref,
|
||||
"exists": exists,
|
||||
"truncated": truncated,
|
||||
}
|
||||
)
|
||||
|
||||
report["constraints"].append(c_rep)
|
||||
|
||||
# decide exit code
|
||||
if report["secrets"]:
|
||||
return 2, report
|
||||
|
||||
if report["missing_files"]:
|
||||
return 2, report
|
||||
|
||||
if report["truncated"] > 0:
|
||||
return 1, report
|
||||
|
||||
return 0, report
|
||||
|
||||
|
||||
def main(argv: List[str]) -> int:
|
||||
import sys
|
||||
|
||||
if len(argv) < 2:
|
||||
print(json.dumps({"error": "manifest path required"}))
|
||||
return 2
|
||||
|
||||
path = argv[1]
|
||||
base_dir = argv[2] if len(argv) > 2 else None
|
||||
|
||||
code, report = validate_manifest(path, base_dir=base_dir)
|
||||
print(json.dumps(report))
|
||||
return code
|
||||
|
||||
|
||||
# no execution at import time
|
||||
@@ -1,56 +0,0 @@
|
||||
"""Command-line wrapper around src.validators.mindmodel_validator.validate_manifest
|
||||
|
||||
This tiny CLI loads a manifest and writes a structured JSON report to stdout
|
||||
and optionally to a file path. It is report-only: it never raises an error or
|
||||
changes exit code based on findings.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
def _write_report(report: dict[str, Any], path: Path | None) -> None:
|
||||
text = json.dumps(report, indent=2, ensure_ascii=False)
|
||||
print(text)
|
||||
if path:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_text(text, encoding="utf-8")
|
||||
|
||||
|
||||
def main(argv: list[str] | None = None) -> int:
|
||||
parser = argparse.ArgumentParser("validate_mindmodel")
|
||||
parser.add_argument("manifest", nargs="?", help="path to manifest file")
|
||||
parser.add_argument("--manifest", dest="manifest_opt", help="path to manifest file")
|
||||
parser.add_argument("--report", help="optional output report path")
|
||||
args = parser.parse_args(argv)
|
||||
|
||||
manifest = args.manifest_opt or args.manifest
|
||||
if not manifest:
|
||||
parser.error("manifest path is required (positional or --manifest)")
|
||||
|
||||
# import here to keep CLI tiny when unused
|
||||
try:
|
||||
from src.validators.mindmodel_validator import validate_manifest
|
||||
except Exception as e: # pragma: no cover - defensive
|
||||
print(f"Failed to import validator: {e}")
|
||||
return 0
|
||||
|
||||
try:
|
||||
report = validate_manifest(manifest, report_only=True)
|
||||
except Exception as e: # never fail the process
|
||||
report = {"error": str(e)}
|
||||
|
||||
report_path = Path(args.report) if args.report else None
|
||||
_write_report(report, report_path)
|
||||
|
||||
# always exit zero for report-only operation
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Reference in New Issue
Block a user