all in
This commit is contained in:
14
.claude/settings.local.json
Normal file
14
.claude/settings.local.json
Normal file
@@ -0,0 +1,14 @@
|
|||||||
|
{
|
||||||
|
"permissions": {
|
||||||
|
"allow": [
|
||||||
|
"Bash(bash:*)",
|
||||||
|
"Bash(cat /c/dev/obsidian-claw/.superpowers/brainstorm/*/state/server-info)",
|
||||||
|
"Bash(ls -d /c/dev/obsidian-claw/KnowledgeVault/Default/*/)",
|
||||||
|
"Bash(git init:*)",
|
||||||
|
"Bash(git add:*)",
|
||||||
|
"Bash(git commit -m ':*)",
|
||||||
|
"WebFetch(domain:www.ollama.com)",
|
||||||
|
"mcp__web-reader__webReader"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,103 @@
|
|||||||
|
<h2>Architecture Overview</h2>
|
||||||
|
<p class="subtitle">Layered Protocol Model — Obsidian RAG Plugin for OpenClaw</p>
|
||||||
|
|
||||||
|
<div class="section">
|
||||||
|
<h3>System Boundary Diagram</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">Architecture: Obsidian RAG Plugin</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 13px; line-height: 1.6; padding: 20px;">
|
||||||
|
<div style="display: flex; gap: 20px; flex-wrap: wrap;">
|
||||||
|
<!-- User Layer -->
|
||||||
|
<div style="flex: 1; min-width: 200px;">
|
||||||
|
<div style="background: #1a1a2e; border: 2px solid #e94560; border-radius: 8px; padding: 12px; margin-bottom: 10px;">
|
||||||
|
<div style="color: #e94560; font-weight: bold; margin-bottom: 6px;">USER LAYER</div>
|
||||||
|
<div style="color: #eee;">Natural Language Query</div>
|
||||||
|
<div style="color: #888; font-size: 11px;">"How was my mental health in 2024?"</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
<!-- OpenClaw Agent -->
|
||||||
|
<div style="flex: 1; min-width: 220px;">
|
||||||
|
<div style="background: #1a1a2e; border: 2px solid #0f3460; border-radius: 8px; padding: 12px; margin-bottom: 10px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 6px;">OPENCLAW AGENT</div>
|
||||||
|
<div style="color: #eee;">Tool Router</div>
|
||||||
|
<div style="color: #888; font-size: 11px;">Selects plugin tools based on intent</div>
|
||||||
|
<div style="color: #eee; margin-top: 4px;">Session Manager</div>
|
||||||
|
<div style="color: #888; font-size: 11px;">Tracks plugin health state</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="margin: 16px 0; text-align: center; color: #666; font-size: 20px;">↕ Tool Calls & Results</div>
|
||||||
|
|
||||||
|
<!-- Plugin Layer -->
|
||||||
|
<div style="background: #16213e; border: 2px solid #53a8b6; border-radius: 8px; padding: 12px; margin-bottom: 10px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 8px;">PLUGIN LAYER (TypeScript)</div>
|
||||||
|
<div style="display: flex; gap: 10px; flex-wrap: wrap;">
|
||||||
|
<div style="background: #1a1a2e; border: 1px solid #53a8b6; border-radius: 6px; padding: 8px; flex: 1; min-width: 120px;">
|
||||||
|
<div style="color: #53a8b6; font-size: 11px;">TRANSPORT</div>
|
||||||
|
<div style="color: #eee; font-size: 12px;">Tool Registry</div>
|
||||||
|
<div style="color: #eee; font-size: 12px;">Error Normalizer</div>
|
||||||
|
</div>
|
||||||
|
<div style="background: #1a1a2e; border: 1px solid #53a8b6; border-radius: 6px; padding: 8px; flex: 1; min-width: 120px;">
|
||||||
|
<div style="color: #53a8b6; font-size: 11px;">SESSION</div>
|
||||||
|
<div style="color: #eee; font-size: 12px;">Vault Watcher</div>
|
||||||
|
<div style="color: #eee; font-size: 12px;">Sync Scheduler</div>
|
||||||
|
</div>
|
||||||
|
<div style="background: #1a1a2e; border: 1px solid #53a8b6; border-radius: 6px; padding: 8px; flex: 1; min-width: 120px;">
|
||||||
|
<div style="color: #53a8b6; font-size: 11px;">TOOL</div>
|
||||||
|
<div style="color: #eee; font-size: 12px;">search</div>
|
||||||
|
<div style="color: #eee; font-size: 12px;">index / status</div>
|
||||||
|
<div style="color: #eee; font-size: 12px;">memory_store</div>
|
||||||
|
</div>
|
||||||
|
<div style="background: #1a1a2e; border: 1px solid #53a8b6; border-radius: 6px; padding: 8px; flex: 1; min-width: 120px;">
|
||||||
|
<div style="color: #53a8b6; font-size: 11px;">DATA</div>
|
||||||
|
<div style="color: #eee; font-size: 12px;">LanceDB Client</div>
|
||||||
|
<div style="color: #eee; font-size: 12px;">Indexer Bridge</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="margin: 16px 0; text-align: center; color: #666; font-size: 20px;">↕ CLI & Filesystem</div>
|
||||||
|
|
||||||
|
<!-- Data Layer -->
|
||||||
|
<div style="display: flex; gap: 10px; flex-wrap: wrap;">
|
||||||
|
<div style="background: #1a1a2e; border: 2px solid #e94560; border-radius: 8px; padding: 12px; flex: 1; min-width: 180px;">
|
||||||
|
<div style="color: #e94560; font-weight: bold; margin-bottom: 4px;">DATA LAYER</div>
|
||||||
|
<div style="color: #eee;">LanceDB <span style="color: #888;">(vectors.lance)</span></div>
|
||||||
|
<div style="color: #eee;">Ollama <span style="color: #888;">(localhost:11434)</span></div>
|
||||||
|
<div style="color: #eee;">Obsidian Vault <span style="color: #888;">(filesystem)</span></div>
|
||||||
|
</div>
|
||||||
|
<div style="background: #1a1a2e; border: 2px solid #f0a500; border-radius: 8px; padding: 12px; flex: 1; min-width: 180px;">
|
||||||
|
<div style="color: #f0a500; font-weight: bold; margin-bottom: 4px;">INDEXER (Python CLI)</div>
|
||||||
|
<div style="color: #eee;">obsidian-rag index</div>
|
||||||
|
<div style="color: #eee;">obsidian-rag sync</div>
|
||||||
|
<div style="color: #eee;">obsidian-rag reindex</div>
|
||||||
|
<div style="color: #eee;">obsidian-rag status</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Key Design Decisions</h3>
|
||||||
|
<div class="pros-cons">
|
||||||
|
<div class="pros"><h4>Chosen</h4>
|
||||||
|
<ul>
|
||||||
|
<li>Layered protocol: Transport → Session → Tool → Data</li>
|
||||||
|
<li>Plugin is thin TS wrapper; heavy lifting in Python</li>
|
||||||
|
<li>LanceDB embedded — no server process</li>
|
||||||
|
<li>Vault watcher auto-syncs on file changes</li>
|
||||||
|
<li>Graceful degradation on dependency failures</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="cons"><h4>Rejected</h4>
|
||||||
|
<ul>
|
||||||
|
<li>Event bus (over-engineered for single plugin)</li>
|
||||||
|
<li>Minimal spec (no protocol contract = ad-hoc)</li>
|
||||||
|
<li>Server-based vector DB (unnecessary complexity)</li>
|
||||||
|
<li>On-demand-only sync (user chose auto-sync)</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
150
.superpowers/brainstorm/27-1775849590/content/data-layer.html
Normal file
150
.superpowers/brainstorm/27-1775849590/content/data-layer.html
Normal file
@@ -0,0 +1,150 @@
|
|||||||
|
<h2>Data Layer</h2>
|
||||||
|
<p class="subtitle">LanceDB Schema, Indexer Bridge, Chunking & Embedding Pipeline</p>
|
||||||
|
|
||||||
|
<div class="section">
|
||||||
|
<h3>LanceDB Table Schema</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">obsidian_chunks table</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; padding: 16px; background: #0d1117; color: #c9d1d9;">
|
||||||
|
<pre style="margin: 0;">┌──────────────────┬──────────────┬─────────────────────────────────────┐
|
||||||
|
│ Column │ Type │ Description │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ vector │ FixedList │ 1024-dim float32 embedding │
|
||||||
|
│ │ float32[1024]│ (mxbai-embed-large via Ollama) │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ chunk_id │ string │ UUID v4, primary key │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ chunk_text │ string │ Raw markdown text of the chunk │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ source_file │ string │ Relative path from vault root │
|
||||||
|
│ │ │ e.g. "Journal/2024-01-15.md" │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ source_directory │ string │ Top-level directory name │
|
||||||
|
│ │ │ e.g. "Journal", "Entertainment" │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ section │ string|null │ Section heading (structured notes) │
|
||||||
|
│ │ │ e.g. "#mentalhealth", "#finance" │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ date │ string|null │ ISO 8601 date parsed from filename │
|
||||||
|
│ │ │ e.g. "2024-01-15" │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ tags │ list<string> │ All hashtags in chunk │
|
||||||
|
│ │ │ e.g. ["#mentalhealth", "#therapy"] │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ chunk_index │ int32 │ Position within source document │
|
||||||
|
│ │ │ 0-indexed │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ total_chunks │ int32 │ Total chunks for this source file │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ modified_at │ string │ File mtime, ISO 8601 │
|
||||||
|
│ │ │ Used for incremental sync │
|
||||||
|
├──────────────────┼──────────────┼─────────────────────────────────────┤
|
||||||
|
│ indexed_at │ string │ When this chunk was indexed │
|
||||||
|
│ │ │ ISO 8601 timestamp │
|
||||||
|
└──────────────────┴──────────────┴─────────────────────────────────────┘</pre>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 16px;">
|
||||||
|
<h3>Indexer Bridge: TS ↔ Python</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">How the TypeScript plugin invokes the Python indexer</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; line-height: 1.7; padding: 16px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 8px;">INVOCATION</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">child_process.spawn("obsidian-rag", [mode], { env: { ...config } })</div>
|
||||||
|
<div style="color: #888; margin-left: 12px; margin-bottom: 8px;">Plugin spawns the Python CLI as a subprocess</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 8px;">COMMUNICATION</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">stdin: Not used</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">stdout: NDJSON progress lines (streamed to agent via status)</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">stderr: Error output (logged, surfaced on non-zero exit)</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">exit code: 0 = success, 1 = partial failure, 2 = fatal</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-top: 12px; margin-bottom: 8px;">PROGRESS EVENTS (stdout NDJSON)</div>
|
||||||
|
<div style="background: #0d1117; padding: 10px; border-radius: 6px; color: #c9d1d9; line-height: 1.6;">
|
||||||
|
<div>{"type":"progress","phase":"embedding","current":150,"total":677}</div>
|
||||||
|
<div>{"type":"progress","phase":"storing","current":150,"total":677}</div>
|
||||||
|
<div>{"type":"complete","indexed_files":677,"total_chunks":3421,"duration_ms":45230,"errors":[]}</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-top: 12px; margin-bottom: 8px;">SYNC RESULT FILE</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">Written to ~/.obsidian-rag/sync-result.json on completion</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">Read by plugin on next tool call to update health state</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">Also serves as the source for status tool responses</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 16px;">
|
||||||
|
<h3>Chunking Pipeline</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">From markdown file → embeddable chunks</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; line-height: 1.7; padding: 16px;">
|
||||||
|
<div style="display: flex; flex-direction: column; gap: 4px;">
|
||||||
|
<div style="display: flex; align-items: center; gap: 8px;">
|
||||||
|
<div style="background: #1a472a; color: #2ecc71; padding: 2px 8px; border-radius: 4px; font-size: 11px;">1. SCAN</div>
|
||||||
|
<div style="color: #eee;">Walk vault dirs (respect deny/allow lists) → collect *.md files</div>
|
||||||
|
</div>
|
||||||
|
<div style="color: #666; margin-left: 40px;">↓</div>
|
||||||
|
<div style="display: flex; align-items: center; gap: 8px;">
|
||||||
|
<div style="background: #1a472a; color: #2ecc71; padding: 2px 8px; border-radius: 4px; font-size: 11px;">2. PARSE</div>
|
||||||
|
<div style="color: #eee;">Extract frontmatter, headings, tags, dates from filename/path</div>
|
||||||
|
</div>
|
||||||
|
<div style="color: #666; margin-left: 40px;">↓</div>
|
||||||
|
<div style="display: flex; align-items: center; gap: 8px;">
|
||||||
|
<div style="background: #3d3200; color: #f0a500; padding: 2px 8px; border-radius: 4px; font-size: 11px;">3. CHUNK</div>
|
||||||
|
<div style="color: #eee;">Structured notes → split by section headers</div>
|
||||||
|
<div style="color: #eee;">Unstructured → sliding window (500 tok, 100 overlap)</div>
|
||||||
|
</div>
|
||||||
|
<div style="color: #666; margin-left: 40px;">↓</div>
|
||||||
|
<div style="display: flex; align-items: center; gap: 8px;">
|
||||||
|
<div style="background: #3d3200; color: #f0a500; padding: 2px 8px; border-radius: 4px; font-size: 11px;">4. ENRICH</div>
|
||||||
|
<div style="color: #eee;">Attach metadata: source_file, section, date, tags, chunk_index</div>
|
||||||
|
</div>
|
||||||
|
<div style="color: #666; margin-left: 40px;">↓</div>
|
||||||
|
<div style="display: flex; align-items: center; gap: 8px;">
|
||||||
|
<div style="background: #1a3a5c; color: #53a8b6; padding: 2px 8px; border-radius: 4px; font-size: 11px;">5. EMBED</div>
|
||||||
|
<div style="color: #eee;">Batch chunks → Ollama /api/embed (mxbai-embed-large, 1024-dim)</div>
|
||||||
|
<div style="color: #888;">Batch size: 64 chunks per request</div>
|
||||||
|
</div>
|
||||||
|
<div style="color: #666; margin-left: 40px;">↓</div>
|
||||||
|
<div style="display: flex; align-items: center; gap: 8px;">
|
||||||
|
<div style="background: #1a3a5c; color: #53a8b6; padding: 2px 8px; border-radius: 4px; font-size: 11px;">6. STORE</div>
|
||||||
|
<div style="color: #eee;">Write vectors + metadata to LanceDB (upsert by chunk_id)</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-top: 16px; margin-bottom: 6px;">INCREMENTAL SYNC LOGIC</div>
|
||||||
|
<div style="color: #eee; line-height: 1.6;">
|
||||||
|
<div>1. Read last_sync_mtime from sync-result.json</div>
|
||||||
|
<div>2. Walk vault, compare each file's mtime vs last_sync_mtime</div>
|
||||||
|
<div>3. New/modified files → full chunk pipeline</div>
|
||||||
|
<div>4. Deleted files → remove chunks by source_file from LanceDB</div>
|
||||||
|
<div>5. Unchanged files → skip entirely</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Key Design Points</h3>
|
||||||
|
<div class="pros-cons">
|
||||||
|
<div class="pros"><h4>Why this works</h4>
|
||||||
|
<ul>
|
||||||
|
<li>Subprocess bridge is simple — no socket, no HTTP, just spawn+stdout</li>
|
||||||
|
<li>NDJSON progress lets plugin stream indexing status to agent in real-time</li>
|
||||||
|
<li>LanceDB upsert by chunk_id makes incremental sync idempotent</li>
|
||||||
|
<li>Batch embedding (64 chunks) balances throughput vs Ollama memory</li>
|
||||||
|
<li>Two chunking strategies handle both structured and unstructured notes</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="cons"><h4>Trade-offs</h4>
|
||||||
|
<ul>
|
||||||
|
<li>Subprocess spawn adds ~200ms per invocation (acceptable for CLI)</li>
|
||||||
|
<li>sync-result.json is a shared filesystem contract — needs careful locking</li>
|
||||||
|
<li>500-token sliding window is configurable but not adaptive to content</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
@@ -0,0 +1,126 @@
|
|||||||
|
<h2>Security & Privacy</h2>
|
||||||
|
<p class="subtitle">Path Traversal Prevention, Input Sanitization, Sensitive Content Guards & Local-Only Enforcement</p>
|
||||||
|
|
||||||
|
<div class="section">
|
||||||
|
<h3>Security Architecture: Four Layers</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">Defense in depth — each layer blocks what the previous might miss</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 13px; padding: 16px;">
|
||||||
|
<div style="display: flex; flex-direction: column; gap: 10px;">
|
||||||
|
|
||||||
|
<div style="display: flex; gap: 12px; align-items: flex-start;">
|
||||||
|
<div style="background: #1a3a5c; border: 2px solid #53a8b6; border-radius: 8px; padding: 10px; min-width: 80px; text-align: center;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold;">LAYER 1</div>
|
||||||
|
<div style="color: #888; font-size: 10px;">Boundary</div>
|
||||||
|
</div>
|
||||||
|
<div style="flex: 1;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold;">Path Traversal Prevention</div>
|
||||||
|
<div style="color: #eee; margin-top: 4px;">All file reads restricted to vault_path. Reject:</div>
|
||||||
|
<div style="color: #f0a500; margin-left: 12px;">• Any path containing ".." components</div>
|
||||||
|
<div style="color: #f0a500; margin-left: 12px;">• Absolute paths not under vault_path</div>
|
||||||
|
<div style="color: #f0a500; margin-left: 12px;">• Symlinks resolving outside vault_path</div>
|
||||||
|
<div style="color: #eee; margin-top: 4px;">Implementation: <code>path.resolve(vault_path, input)</code> must start with <code>vault_path</code></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="display: flex; gap: 12px; align-items: flex-start;">
|
||||||
|
<div style="background: #3d3200; border: 2px solid #f0a500; border-radius: 8px; padding: 10px; min-width: 80px; text-align: center;">
|
||||||
|
<div style="color: #f0a500; font-weight: bold;">LAYER 2</div>
|
||||||
|
<div style="color: #888; font-size: 10px;">Sanitize</div>
|
||||||
|
</div>
|
||||||
|
<div style="flex: 1;">
|
||||||
|
<div style="color: #f0a500; font-weight: bold;">Input Sanitization</div>
|
||||||
|
<div style="color: #eee; margin-top: 4px;">All vault content treated as untrusted before embedding:</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">• Strip HTML tags (prevent XSS in chunk_text)</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">• Remove executable code blocks (```...```)</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">• Normalize whitespace (prevent injection via formatting)</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">• Cap chunk_text at 2000 chars (prevent oversized payloads)</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="display: flex; gap: 12px; align-items: flex-start;">
|
||||||
|
<div style="background: #3d0000; border: 2px solid #e94560; border-radius: 8px; padding: 10px; min-width: 80px; text-align: center;">
|
||||||
|
<div style="color: #e94560; font-weight: bold;">LAYER 3</div>
|
||||||
|
<div style="color: #888; font-size: 10px;">Guard</div>
|
||||||
|
</div>
|
||||||
|
<div style="flex: 1;">
|
||||||
|
<div style="color: #e94560; font-weight: bold;">Sensitive Content Guard</div>
|
||||||
|
<div style="color: #eee; margin-top: 4px;">Detect and gate sensitive content in search results:</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">• Health: #mentalhealth, #physicalhealth, medication, therapy</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">• Financial: owe, owed, debt, paid, $, spent</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">• Relations: #Relations (configurable section list)</div>
|
||||||
|
<div style="color: #eee; margin-top: 4px;">Action: Set <code style="color: #f0a500;">sensitive_detected=true</code> in response</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">→ Agent must confirm before displaying to user</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">→ Never transmit sensitive content to external APIs</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="display: flex; gap: 12px; align-items: flex-start;">
|
||||||
|
<div style="background: #1a472a; border: 2px solid #2ecc71; border-radius: 8px; padding: 10px; min-width: 80px; text-align: center;">
|
||||||
|
<div style="color: #2ecc71; font-weight: bold;">LAYER 4</div>
|
||||||
|
<div style="color: #888; font-size: 10px;">Network</div>
|
||||||
|
</div>
|
||||||
|
<div style="flex: 1;">
|
||||||
|
<div style="color: #2ecc71; font-weight: bold;">Local-Only Enforcement</div>
|
||||||
|
<div style="color: #eee; margin-top: 4px;">All data stays on the local machine:</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">• Ollama: localhost:11434 only</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">• LanceDB: local filesystem only</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">• Config: <code style="color: #f0a500;">"local_only": true</code> enforced</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">• Network audit test: verify no outbound requests</div>
|
||||||
|
<div style="color: #e94560; margin-top: 4px;">If external embedding endpoint configured:</div>
|
||||||
|
<div style="color: #e94560; margin-left: 12px;">→ Require explicit user confirmation</div>
|
||||||
|
<div style="color: #e94560; margin-left: 12px;">→ Block if sensitive content detected in payload</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Directory Access Control</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">deny_dirs and allow_dirs enforcement</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; line-height: 1.7; padding: 16px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 6px;">DENY LIST (default)</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">.obsidian, .trash, zzz-Archive, .git</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">Always excluded — system dirs, trash, archives, VCS</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-top: 10px; margin-bottom: 6px;">ALLOW LIST (optional)</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">If set: ONLY these directories are indexed</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">If empty: all dirs except deny list are indexed</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">e.g. ["Journal", "Finance"] → only these two</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-top: 10px; margin-bottom: 6px;">FILTER LOGIC (Python indexer)</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">1. If allow_dirs is non-empty → only walk those dirs</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">2. Skip any path matching deny_dirs patterns</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">3. Skip hidden dirs (starting with .)</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-top: 10px; margin-bottom: 6px;">FILTER LOGIC (TS plugin)</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">directory_filter parameter validates against known dirs</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">Reject unknown dirs → prevent probing vault structure</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Key Design Points</h3>
|
||||||
|
<div class="pros-cons">
|
||||||
|
<div class="pros"><h4>Why this works</h4>
|
||||||
|
<ul>
|
||||||
|
<li>Four independent layers — bypass one and the others still protect</li>
|
||||||
|
<li>Sensitive content is flagged, not blocked — agent decides with user</li>
|
||||||
|
<li>Local-only is default; external APIs require explicit opt-in + confirmation</li>
|
||||||
|
<li>Directory controls applied in both Python (indexing) and TS (querying)</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="cons"><h4>Trade-offs</h4>
|
||||||
|
<ul>
|
||||||
|
<li>Sensitive content detection is pattern-based — may have false positives/negatives</li>
|
||||||
|
<li>Stripping code blocks loses technical notes content</li>
|
||||||
|
<li>Network audit test must be run manually — no runtime enforcement</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
115
.superpowers/brainstorm/27-1775849590/content/session-layer.html
Normal file
115
.superpowers/brainstorm/27-1775849590/content/session-layer.html
Normal file
@@ -0,0 +1,115 @@
|
|||||||
|
<h2>Session Layer</h2>
|
||||||
|
<p class="subtitle">Vault Watcher, Auto-Sync Scheduling & Plugin Health State Machine</p>
|
||||||
|
|
||||||
|
<div class="section">
|
||||||
|
<h3>Health State Machine</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">Plugin transitions through three states</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 13px; line-height: 1.6; padding: 16px;">
|
||||||
|
<div style="display: flex; justify-content: center; gap: 20px; flex-wrap: wrap; margin-bottom: 16px;">
|
||||||
|
<div style="background: #1a472a; border: 2px solid #2ecc71; border-radius: 8px; padding: 12px; min-width: 140px; text-align: center;">
|
||||||
|
<div style="color: #2ecc71; font-weight: bold; font-size: 14px;">HEALTHY</div>
|
||||||
|
<div style="color: #aaa; font-size: 11px;">All deps up</div>
|
||||||
|
<div style="color: #2ecc71; font-size: 11px;">→ full service</div>
|
||||||
|
</div>
|
||||||
|
<div style="color: #666; font-size: 24px; display: flex; align-items: center;">⇄</div>
|
||||||
|
<div style="background: #3d3200; border: 2px solid #f0a500; border-radius: 8px; padding: 12px; min-width: 140px; text-align: center;">
|
||||||
|
<div style="color: #f0a500; font-weight: bold; font-size: 14px;">DEGRADED</div>
|
||||||
|
<div style="color: #aaa; font-size: 11px;">Ollama down</div>
|
||||||
|
<div style="color: #f0a500; font-size: 11px;">→ stale results ok</div>
|
||||||
|
</div>
|
||||||
|
<div style="color: #666; font-size: 24px; display: flex; align-items: center;">⇄</div>
|
||||||
|
<div style="background: #3d0000; border: 2px solid #e94560; border-radius: 8px; padding: 12px; min-width: 140px; text-align: center;">
|
||||||
|
<div style="color: #e94560; font-weight: bold; font-size: 14px;">UNAVAILABLE</div>
|
||||||
|
<div style="color: #aaa; font-size: 11px;">No index / DB</div>
|
||||||
|
<div style="color: #e94560; font-size: 11px;">→ cannot serve</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="border-top: 1px solid #333; padding-top: 12px; color: #aaa; font-size: 12px;">
|
||||||
|
<div><span style="color: #2ecc71;">HEALTHY → DEGRADED</span>: Ollama health check fails (every 30s)</div>
|
||||||
|
<div><span style="color: #f0a500;">DEGRADED → HEALTHY</span>: Ollama responds again</div>
|
||||||
|
<div><span style="color: #2ecc71;">HEALTHY → UNAVAILABLE</span>: Index deleted or DB corrupted</div>
|
||||||
|
<div><span style="color: #e94560;">UNAVAILABLE → HEALTHY</span>: Successful reindex after recovery</div>
|
||||||
|
<div><span style="color: #f0a500;">DEGRADED → UNAVAILABLE</span>: Index corrupted while Ollama was down</div>
|
||||||
|
<div><span style="color: #e94560;">UNAVAILABLE → DEGRADED</span>: Index restored but Ollama still down</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Vault Watcher: Auto-Sync Protocol</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">File change detection → incremental sync</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 13px; line-height: 1.7; padding: 16px;">
|
||||||
|
<div style="color: #53a8b6;">DETECTION</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">chokidar watches vault_path (respects deny_dirs / allow_dirs)</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">Debounces: 2s after last change event</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">→ Avoids triggering sync for every keystroke in an editor</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; margin-top: 12px;">BATCHING</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">Collector window: 5s (configurable)</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">Groups add/update/delete events into a changeset</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">→ Single sync call for 10 files changed at once</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; margin-top: 12px;">SYNC TRIGGER</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">After debounce + collect: spawn indexer with <code style="color: #f0a500;">obsidian-rag sync</code></div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">Indexer processes only modified files (uses mtime comparison)</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">On completion: indexer writes status JSON to <code style="color: #f0a500;">~/.obsidian-rag/sync-result.json</code></div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; margin-top: 12px;">SYNC RESULT</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">Plugin reads sync-result.json on next tool call</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">Updates health state: HEALTHY if indexed > 0 docs, UNAVAILABLE if 0</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">Stale flag: if last sync > 1h ago and vault changed, next search returns <code style="color: #f0a500;">status: "degraded"</code></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Session Lifecycle</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">What happens from plugin load to shutdown</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 13px; padding: 16px;">
|
||||||
|
<div style="display: flex; flex-direction: column; gap: 8px;">
|
||||||
|
<div style="display: flex; align-items: center; gap: 10px;">
|
||||||
|
<div style="background: #53a8b6; color: #000; padding: 2px 8px; border-radius: 4px; font-weight: bold; font-size: 11px;">ON LOAD</div>
|
||||||
|
<div style="color: #eee;">Read config → Probe Ollama → Probe LanceDB → Probe Vault → Set health state</div>
|
||||||
|
</div>
|
||||||
|
<div style="display: flex; align-items: center; gap: 10px;">
|
||||||
|
<div style="background: #53a8b6; color: #000; padding: 2px 8px; border-radius: 4px; font-weight: bold; font-size: 11px;">REGISTER</div>
|
||||||
|
<div style="color: #eee;">Register 4 tools with OpenClaw → Start vault watcher</div>
|
||||||
|
</div>
|
||||||
|
<div style="display: flex; align-items: center; gap: 10px;">
|
||||||
|
<div style="background: #53a8b6; color: #000; padding: 2px 8px; border-radius: 4px; font-weight: bold; font-size: 11px;">SERVE</div>
|
||||||
|
<div style="color: #eee;">Handle tool calls → Auto-sync on file changes → Health re-probe every 30s</div>
|
||||||
|
</div>
|
||||||
|
<div style="display: flex; align-items: center; gap: 10px;">
|
||||||
|
<div style="background: #e94560; color: #000; padding: 2px 8px; border-radius: 4px; font-weight: bold; font-size: 11px;">ON SHUTDOWN</div>
|
||||||
|
<div style="color: #eee;">Stop vault watcher → Wait for in-flight sync → Clean exit</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Key Design Points</h3>
|
||||||
|
<div class="pros-cons">
|
||||||
|
<div class="pros"><h4>Why this works</h4>
|
||||||
|
<ul>
|
||||||
|
<li>Health state machine is simple — only 3 states, clear transitions</li>
|
||||||
|
<li>Debounce + batch prevents Ollama hammering during rapid edits</li>
|
||||||
|
<li>Stale flag on old sync lets agent warn user about freshness</li>
|
||||||
|
<li>Sync results written to filesystem — no IPC needed between indexer and plugin</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="cons"><h4>Trade-offs</h4>
|
||||||
|
<ul>
|
||||||
|
<li>Vault watcher uses chokidar — filesystem event reliability varies by OS</li>
|
||||||
|
<li>30s health re-probe adds periodic Ollama load</li>
|
||||||
|
<li>Filesystem-based sync result is a polling model — up to 30s stale</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
@@ -0,0 +1,147 @@
|
|||||||
|
<h2>Testing Strategy</h2>
|
||||||
|
<p class="subtitle">Python Tests, TypeScript Tests, Integration Tests & Security Test Suites</p>
|
||||||
|
|
||||||
|
<div class="section">
|
||||||
|
<h3>Test Architecture: Four Suites</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">Each suite tests a distinct concern</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; padding: 16px; display: flex; gap: 16px; flex-wrap: wrap;">
|
||||||
|
<div style="flex: 1; min-width: 220px; background: #1a472a; border: 1px solid #2ecc71; border-radius: 8px; padding: 12px;">
|
||||||
|
<div style="color: #2ecc71; font-weight: bold; margin-bottom: 8px;">PYTHON UNIT</div>
|
||||||
|
<div style="color: #aaa; font-size: 10px; margin-bottom: 6px;">pytest / python/tests/</div>
|
||||||
|
<div style="color: #eee; line-height: 1.6;">
|
||||||
|
<div>test_chunker.py</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Section splitting, sliding window, metadata</div>
|
||||||
|
<div>test_embedder.py</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Mocked Ollama, batch embed, error handling</div>
|
||||||
|
<div>test_vector_store.py</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ LanceDB CRUD, upsert, delete by source</div>
|
||||||
|
<div>test_security.py</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Path traversal, sanitization, sensitive detection</div>
|
||||||
|
<div>test_indexer.py</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Full pipeline, incremental sync, config</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="flex: 1; min-width: 220px; background: #1a3a5c; border: 1px solid #53a8b6; border-radius: 8px; padding: 12px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 8px;">TYPESCRIPT UNIT</div>
|
||||||
|
<div style="color: #aaa; font-size: 10px; margin-bottom: 6px;">vitest / tests/</div>
|
||||||
|
<div style="color: #eee; line-height: 1.6;">
|
||||||
|
<div>search.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Parameter validation, filter logic, response shape</div>
|
||||||
|
<div>index.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Mode validation, subprocess spawn, progress parse</div>
|
||||||
|
<div>memory.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Key/value storage, auto-suggest patterns</div>
|
||||||
|
<div>vault-watcher.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Chokidar events, debounce, batching</div>
|
||||||
|
<div>security-guard.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Directory filter validation, sensitive flag</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="flex: 1; min-width: 220px; background: #3d3200; border: 1px solid #f0a500; border-radius: 8px; padding: 12px;">
|
||||||
|
<div style="color: #f0a500; font-weight: bold; margin-bottom: 8px;">INTEGRATION</div>
|
||||||
|
<div style="color: #aaa; font-size: 10px; margin-bottom: 6px;">E2E / tests/integration/</div>
|
||||||
|
<div style="color: #eee; line-height: 1.6;">
|
||||||
|
<div>full_pipeline.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Index vault → search → verify results</div>
|
||||||
|
<div>sync_cycle.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Modify file → auto-sync → search updated</div>
|
||||||
|
<div>health_state.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Stop Ollama → degraded → restart → healthy</div>
|
||||||
|
<div>openclaw_protocol.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Agent calls tools, validates envelope</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div style="flex: 1; min-width: 220px; background: #3d0000; border: 1px solid #e94560; border-radius: 8px; padding: 12px;">
|
||||||
|
<div style="color: #e94560; font-weight: bold; margin-bottom: 8px;">SECURITY</div>
|
||||||
|
<div style="color: #aaa; font-size: 10px; margin-bottom: 6px;">Dedicated / tests/security/</div>
|
||||||
|
<div style="color: #eee; line-height: 1.6;">
|
||||||
|
<div>path_traversal.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ ../, symlinks, absolute, encoded paths</div>
|
||||||
|
<div>xss_prevention.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ HTML/script injection in chunk text</div>
|
||||||
|
<div>prompt_injection.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Malicious content in vault notes</div>
|
||||||
|
<div>network_audit.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Verify zero outbound when local_only=true</div>
|
||||||
|
<div>sensitive_content.test.ts</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">→ Pattern detection, flagging, blocking</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Testing Approach by Layer</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">What gets mocked vs what gets hit for real</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; padding: 16px;">
|
||||||
|
<table style="width: 100%; border-collapse: collapse; color: #eee;">
|
||||||
|
<thead>
|
||||||
|
<tr style="border-bottom: 2px solid #53a8b6;">
|
||||||
|
<th style="text-align: left; padding: 6px;">Component</th>
|
||||||
|
<th style="text-align: left; padding: 6px;">Unit Test</th>
|
||||||
|
<th style="text-align: left; padding: 6px;">Integration Test</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">Ollama embedding</td>
|
||||||
|
<td style="padding: 6px; color: #f0a500;">Mocked — fixed 1024-dim vectors</td>
|
||||||
|
<td style="padding: 6px; color: #2ecc71;">Real — requires Ollama running</td>
|
||||||
|
</tr>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">LanceDB</td>
|
||||||
|
<td style="padding: 6px; color: #2ecc71;">Real — temp directory, cleaned up</td>
|
||||||
|
<td style="padding: 6px; color: #2ecc71;">Real — temp directory, cleaned up</td>
|
||||||
|
</tr>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">Obsidian vault</td>
|
||||||
|
<td style="padding: 6px; color: #f0a500;">Mocked — fixture markdown files</td>
|
||||||
|
<td style="padding: 6px; color: #2ecc71;">Real — temp vault with real files</td>
|
||||||
|
</tr>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">Python CLI</td>
|
||||||
|
<td style="padding: 6px; color: #f0a500;">Mocked subprocess</td>
|
||||||
|
<td style="padding: 6px; color: #2ecc71;">Real — actual CLI invocation</td>
|
||||||
|
</tr>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">Chokidar watcher</td>
|
||||||
|
<td style="padding: 6px; color: #f0a500;">Mocked events</td>
|
||||||
|
<td style="padding: 6px; color: #2ecc71;">Real — actual file system events</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td style="padding: 6px;">OpenClaw agent</td>
|
||||||
|
<td style="padding: 6px; color: #888;">N/A</td>
|
||||||
|
<td style="padding: 6px; color: #2ecc71;">Real — tool call envelope validation</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Key Design Points</h3>
|
||||||
|
<div class="pros-cons">
|
||||||
|
<div class="pros"><h4>Why this works</h4>
|
||||||
|
<ul>
|
||||||
|
<li>Four suites cover all four security layers + both languages</li>
|
||||||
|
<li>LanceDB uses real temp dirs — no mock drift for vector operations</li>
|
||||||
|
<li>Integration tests require Ollama but skip gracefully if unavailable</li>
|
||||||
|
<li>Security suite is standalone — always runs, no dependencies</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="cons"><h4>Trade-offs</h4>
|
||||||
|
<ul>
|
||||||
|
<li>Integration tests need Ollama installed locally</li>
|
||||||
|
<li>File watcher integration tests are timing-dependent (debounce/batch)</li>
|
||||||
|
<li>Sensitive content detection is pattern-based — test coverage is only as good as patterns</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
170
.superpowers/brainstorm/27-1775849590/content/tool-layer.html
Normal file
170
.superpowers/brainstorm/27-1775849590/content/tool-layer.html
Normal file
@@ -0,0 +1,170 @@
|
|||||||
|
<h2>Tool Layer</h2>
|
||||||
|
<p class="subtitle">Four Plugin Tools, Their Contracts & the OpenClaw Interaction Protocol</p>
|
||||||
|
|
||||||
|
<div class="section">
|
||||||
|
<h3>Tool 1: obsidian_rag_search</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">Primary semantic search tool</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; padding: 16px; display: flex; gap: 20px; flex-wrap: wrap;">
|
||||||
|
<div style="flex: 1; min-width: 240px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 6px;">PARAMETERS</div>
|
||||||
|
<div style="color: #eee;">query <span style="color: #e94560;">(required)</span> string</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">Natural language question</div>
|
||||||
|
<div style="color: #eee;">max_results <span style="color: #888;">(opt)</span> int, default 5</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">Max chunks to return (1-50)</div>
|
||||||
|
<div style="color: #eee;">directory_filter <span style="color: #888;">(opt)</span> string[]</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">Limit to subdirs e.g. ["Journal"]</div>
|
||||||
|
<div style="color: #eee;">date_range <span style="color: #888;">(opt)</span> {from, to}</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">ISO 8601 dates</div>
|
||||||
|
<div style="color: #eee;">tags <span style="color: #888;">(opt)</span> string[]</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">Filter by hashtags</div>
|
||||||
|
</div>
|
||||||
|
<div style="flex: 1; min-width: 240px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 6px;">RESPONSE data</div>
|
||||||
|
<div style="color: #eee;">results: [{</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">chunk_text: string</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">score: number (0-1)</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">source_file: string</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">section: string | null</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">date: string | null</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">tags: string[]</div>
|
||||||
|
<div style="color: #eee; margin-left: 12px;">chunk_index: number</div>
|
||||||
|
<div style="color: #eee;">}]</div>
|
||||||
|
<div style="color: #eee; margin-top: 6px;">sensitive_detected: boolean</div>
|
||||||
|
<div style="color: #888; margin-left: 12px;">If true, agent should confirm before display</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 16px;">
|
||||||
|
<h3>Tool 2: obsidian_rag_index</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">Trigger indexing from within OpenClaw</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; padding: 16px; display: flex; gap: 20px; flex-wrap: wrap;">
|
||||||
|
<div style="flex: 1; min-width: 240px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 6px;">PARAMETERS</div>
|
||||||
|
<div style="color: #eee;">mode <span style="color: #e94560;">(required)</span> enum</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">"full" | "sync" | "reindex"</div>
|
||||||
|
<div style="color: #eee;">full: initial index of entire vault</div>
|
||||||
|
<div style="color: #eee;">sync: incremental (mtime diff only)</div>
|
||||||
|
<div style="color: #eee;">reindex: nuke + rebuild from scratch</div>
|
||||||
|
</div>
|
||||||
|
<div style="flex: 1; min-width: 240px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 6px;">RESPONSE data</div>
|
||||||
|
<div style="color: #eee;">indexed_files: number</div>
|
||||||
|
<div style="color: #eee;">total_chunks: number</div>
|
||||||
|
<div style="color: #eee;">duration_ms: number</div>
|
||||||
|
<div style="color: #eee;">errors: [{file, message}]</div>
|
||||||
|
<div style="color: #888; margin-top: 8px;">Long-running: returns immediately with</div>
|
||||||
|
<div style="color: #888;">job_id, agent polls via status tool</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 16px;">
|
||||||
|
<h3>Tool 3: obsidian_rag_status</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">Index health and sync state</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; padding: 16px; display: flex; gap: 20px; flex-wrap: wrap;">
|
||||||
|
<div style="flex: 1; min-width: 240px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 6px;">PARAMETERS</div>
|
||||||
|
<div style="color: #888;">(none — always returns current state)</div>
|
||||||
|
</div>
|
||||||
|
<div style="flex: 1; min-width: 240px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 6px;">RESPONSE data</div>
|
||||||
|
<div style="color: #eee;">plugin_health: "healthy"|"degraded"|"unavailable"</div>
|
||||||
|
<div style="color: #eee;">total_docs: number</div>
|
||||||
|
<div style="color: #eee;">total_chunks: number</div>
|
||||||
|
<div style="color: #eee;">last_sync: string (ISO 8601)</div>
|
||||||
|
<div style="color: #eee;">unindexed_files: number</div>
|
||||||
|
<div style="color: #eee;">ollama_status: "up" | "down"</div>
|
||||||
|
<div style="color: #eee;">active_job: {id, mode, progress} | null</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 16px;">
|
||||||
|
<h3>Tool 4: obsidian_rag_memory_store</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">Commit facts to OpenClaw memory for faster retrieval</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; padding: 16px; display: flex; gap: 20px; flex-wrap: wrap;">
|
||||||
|
<div style="flex: 1; min-width: 240px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 6px;">PARAMETERS</div>
|
||||||
|
<div style="color: #eee;">key <span style="color: #e94560;">(required)</span> string</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">Identifier for the fact</div>
|
||||||
|
<div style="color: #eee;">value <span style="color: #e94560;">(required)</span> string</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">The fact to remember</div>
|
||||||
|
<div style="color: #eee;">source <span style="color: #e94560;">(required)</span> string</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">Source file path in vault</div>
|
||||||
|
</div>
|
||||||
|
<div style="flex: 1; min-width: 240px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 6px;">RESPONSE data</div>
|
||||||
|
<div style="color: #eee;">stored: boolean</div>
|
||||||
|
<div style="color: #eee;">key: string (echoed back)</div>
|
||||||
|
<div style="color: #888; margin-top: 8px;">Auto-suggest: When search results</div>
|
||||||
|
<div style="color: #888;">match financial/health/commitment</div>
|
||||||
|
<div style="color: #888;">patterns, plugin flags</div>
|
||||||
|
<div style="color: #888;">sensitive_detected=true and includes</div>
|
||||||
|
<div style="color: #888;">memory_suggestion with key/value</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>OpenClaw Interaction Protocol</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">How the agent decides which tool to use and when</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; padding: 16px;">
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-bottom: 8px;">INTENT → TOOL MAPPING</div>
|
||||||
|
<table style="width: 100%; border-collapse: collapse; color: #eee;">
|
||||||
|
<thead>
|
||||||
|
<tr style="border-bottom: 2px solid #53a8b6;">
|
||||||
|
<th style="text-align: left; padding: 6px;">User Intent Pattern</th>
|
||||||
|
<th style="text-align: left; padding: 6px;">Tool</th>
|
||||||
|
<th style="text-align: left; padding: 6px;">Agent Behavior</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">"What did I write about X?"<br>"How was my Y in Z?"</td>
|
||||||
|
<td style="padding: 6px; color: #f0a500;">search</td>
|
||||||
|
<td style="padding: 6px;">Query vault, synthesize answer from chunks</td>
|
||||||
|
</tr>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">"Index my vault"<br>"Update the search index"</td>
|
||||||
|
<td style="padding: 6px; color: #f0a500;">index</td>
|
||||||
|
<td style="padding: 6px;">Choose mode (sync default, full on no index), poll status</td>
|
||||||
|
</tr>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">"Is the index up to date?"<br>"How many docs are indexed?"</td>
|
||||||
|
<td style="padding: 6px; color: #f0a500;">status</td>
|
||||||
|
<td style="padding: 6px;">Report health, suggest index if stale</td>
|
||||||
|
</tr>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">"Remember this"<br>Auto-suggest after search</td>
|
||||||
|
<td style="padding: 6px; color: #f0a500;">memory_store</td>
|
||||||
|
<td style="padding: 6px;">Confirm with user, store key+value+source</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; font-weight: bold; margin-top: 16px; margin-bottom: 8px;">PROTOCOL SEQUENCE</div>
|
||||||
|
<div style="color: #eee; line-height: 1.8;">
|
||||||
|
<div>1. Agent receives user query with vault-related intent</div>
|
||||||
|
<div>2. Agent calls <span style="color: #f0a500;">obsidian_rag_status</span> (if first interaction or >5min since last check)</div>
|
||||||
|
<div style="margin-left: 24px; color: #888;">→ Skips if status already cached and healthy</div>
|
||||||
|
<div>3. If unavailable → agent suggests running <span style="color: #f0a500;">obsidian_rag_index</span> (mode: "full")</div>
|
||||||
|
<div>4. If degraded → agent warns user but proceeds with search</div>
|
||||||
|
<div>5. Agent calls <span style="color: #f0a500;">obsidian_rag_search</span> with appropriate filters</div>
|
||||||
|
<div>6. If <span style="color: #f0a500;">sensitive_detected=true</span> → agent confirms before displaying</div>
|
||||||
|
<div>7. Agent synthesizes answer from chunks + existing knowledge</div>
|
||||||
|
<div>8. If memory_suggestion present → agent asks if user wants to store</div>
|
||||||
|
<div style="margin-left: 24px; color: #888;">→ User confirms → <span style="color: #f0a500;">obsidian_rag_memory_store</span></div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
<h2>Transport Layer</h2>
|
||||||
|
<p class="subtitle">Tool Registration, Request/Response Contracts & Error Normalization</p>
|
||||||
|
|
||||||
|
<div class="section">
|
||||||
|
<h3>Tool Registration Flow</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">Plugin Lifecycle: Install → Register → Serve</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 13px; line-height: 1.8; padding: 16px;">
|
||||||
|
<div style="color: #53a8b6;">1. INSTALL</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">openclaw plugins install clawhub:obsidian-rag</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">→ Downloads plugin, reads openclaw.plugin.json</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; margin-top: 12px;">2. REGISTER</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">Plugin.register(tools) → OpenClaw Tool Registry</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">→ 4 tools registered: search, index, status, memory_store</div>
|
||||||
|
<div style="color: #888; margin-left: 16px;">→ Each tool declares: name, description, parameterSchema, requiredPermissions</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; margin-top: 12px;">3. SERVE</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">OpenClaw Agent → tool_call(obsidian_rag_search, {query: "..."}) → Plugin</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">Plugin → structured response → OpenClaw Agent</div>
|
||||||
|
|
||||||
|
<div style="color: #53a8b6; margin-top: 12px;">4. HEALTH CHECK</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">Plugin.onLoad() → probe Ollama, probe LanceDB, probe Vault</div>
|
||||||
|
<div style="color: #eee; margin-left: 16px;">→ Reports status: healthy | degraded | unavailable</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Standardized Response Envelope</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">All tool responses use this envelope</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; line-height: 1.5; padding: 16px; background: #0d1117; color: #c9d1d9;">
|
||||||
|
<pre style="margin: 0;">{
|
||||||
|
"status": "healthy" | "degraded" | "unavailable",
|
||||||
|
"data": T | null,
|
||||||
|
"error": {
|
||||||
|
"code": string, // e.g. "OLLAMA_UNREACHABLE"
|
||||||
|
"message": string, // human-readable
|
||||||
|
"recoverable": bool, // can the agent retry?
|
||||||
|
"suggestion": string // e.g. "Run obsidian-rag index first"
|
||||||
|
} | null,
|
||||||
|
"meta": {
|
||||||
|
"query_time_ms": number,
|
||||||
|
"chunks_scanned": number,
|
||||||
|
"index_version": string,
|
||||||
|
"vault_mtime": string // ISO 8601
|
||||||
|
}
|
||||||
|
}</pre>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Error Normalization Matrix</h3>
|
||||||
|
<div class="mockup">
|
||||||
|
<div class="mockup-header">How each failure maps to status + error</div>
|
||||||
|
<div class="mockup-body" style="font-family: monospace; font-size: 12px; padding: 16px;">
|
||||||
|
<table style="width: 100%; border-collapse: collapse; color: #eee;">
|
||||||
|
<thead>
|
||||||
|
<tr style="border-bottom: 2px solid #53a8b6;">
|
||||||
|
<th style="text-align: left; padding: 6px;">Failure</th>
|
||||||
|
<th style="text-align: left; padding: 6px;">Status</th>
|
||||||
|
<th style="text-align: left; padding: 6px;">Code</th>
|
||||||
|
<th style="text-align: left; padding: 6px;">Recoverable</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">Ollama down</td>
|
||||||
|
<td style="padding: 6px; color: #f0a500;">degraded</td>
|
||||||
|
<td style="padding: 6px;">OLLAMA_UNREACHABLE</td>
|
||||||
|
<td style="padding: 6px;">yes — retry after start</td>
|
||||||
|
</tr>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">Empty vault / no index</td>
|
||||||
|
<td style="padding: 6px; color: #e94560;">unavailable</td>
|
||||||
|
<td style="padding: 6px;">INDEX_NOT_FOUND</td>
|
||||||
|
<td style="padding: 6px;">yes — run index first</td>
|
||||||
|
</tr>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">LanceDB corrupted</td>
|
||||||
|
<td style="padding: 6px; color: #e94560;">unavailable</td>
|
||||||
|
<td style="padding: 6px;">INDEX_CORRUPTED</td>
|
||||||
|
<td style="padding: 6px;">yes — run reindex</td>
|
||||||
|
</tr>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">Path traversal attempt</td>
|
||||||
|
<td style="padding: 6px; color: #e94560;">unavailable</td>
|
||||||
|
<td style="padding: 6px;">SECURITY_VIOLATION</td>
|
||||||
|
<td style="padding: 6px;">no</td>
|
||||||
|
</tr>
|
||||||
|
<tr style="border-bottom: 1px solid #333;">
|
||||||
|
<td style="padding: 6px;">Sensitive content blocked</td>
|
||||||
|
<td style="padding: 6px; color: #f0a500;">degraded</td>
|
||||||
|
<td style="padding: 6px;">SENSITIVE_FILTERED</td>
|
||||||
|
<td style="padding: 6px;">yes — with confirmation</td>
|
||||||
|
</tr>
|
||||||
|
<tr>
|
||||||
|
<td style="padding: 6px;">Indexer CLI crash</td>
|
||||||
|
<td style="padding: 6px; color: #e94560;">unavailable</td>
|
||||||
|
<td style="padding: 6px;">INDEXER_FAILED</td>
|
||||||
|
<td style="padding: 6px;">yes — retry once</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div class="section" style="margin-top: 20px;">
|
||||||
|
<h3>Key Design Points</h3>
|
||||||
|
<div class="pros-cons">
|
||||||
|
<div class="pros"><h4>Why this works</h4>
|
||||||
|
<ul>
|
||||||
|
<li>OpenClaw agent never sees raw exceptions — all errors normalized</li>
|
||||||
|
<li>Three-state health (healthy/degraded/unavailable) lets agent adapt behavior</li>
|
||||||
|
<li>Recoverable flag lets agent decide whether to retry or inform user</li>
|
||||||
|
<li>Suggestion field gives agent context to offer remediation</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
<div class="cons"><h4>Trade-offs</h4>
|
||||||
|
<ul>
|
||||||
|
<li>Envelope adds ~200 bytes per response (negligible)</li>
|
||||||
|
<li>Agent must handle three status states, not just success/fail</li>
|
||||||
|
<li>Health check on load adds ~1s startup latency</li>
|
||||||
|
</ul>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"reason":"idle timeout","timestamp":1775855591203}
|
||||||
1
.superpowers/brainstorm/27-1775849590/state/server.pid
Normal file
1
.superpowers/brainstorm/27-1775849590/state/server.pid
Normal file
@@ -0,0 +1 @@
|
|||||||
|
27
|
||||||
Reference in New Issue
Block a user