Sprint 0-1: Python indexer, TS plugin scaffolding, and test suite
## What's new **Python indexer (`python/obsidian_rag/`)** — full pipeline from scan to LanceDB: - `config.py` — JSON config loader with cross-platform path resolution - `security.py` — path traversal prevention, HTML stripping, sensitive content detection, dir allow/deny lists - `chunker.py` — section-split for journal entries (date-named files), sliding-window for unstructured notes - `embedder.py` — Ollama `/api/embeddings` client with batched requests and timeout/error handling - `vector_store.py` — LanceDB schema, upsert (merge_insert), delete, search with filters, stats - `indexer.py` — full/sync/reindex pipeline orchestrator with progress yields - `cli.py` — `index | sync | reindex | status` CLI commands **TypeScript plugin (`src/`)** — OpenClaw plugin scaffold: - `utils/` — config loader, TypeScript types, response envelope factory, LanceDB client - `services/` — health state machine (HEALTHY/DEGRADED/UNAVAILABLE), vault watcher with debounce/batching, indexer bridge (subprocess spawner) - `tools/` — 4 tool stubs: search, index, status, memory_store (OpenClaw wiring pending) - `index.ts` — plugin entry point with health probe + vault watcher startup **Config** (`obsidian-rag/config.json`, `openclaw.plugin.json`): - 627 files / 3764 chunks indexed in dev vault **Tests: 76 passing** - Python: 64 pytest tests (chunker, security, vector_store, config) - TypeScript: 12 vitest tests (lancedb client, response envelope) ## Bugs fixed - LanceDB `tags` column filter: `LIKE '%tag%'` → `list_contains(tags, 'tag')` (List<String> column) - LanceDB JS `db.list_tables()` returns `ListTablesResponse` object, not plain array - LanceDB JS result score field: `_score` → `_distance` - TypeScript regex literal with unescaped `/` in path-resolve regex - Python: `create_table_if_not_exists` identity check → name comparison Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
130
src/services/health.ts
Normal file
130
src/services/health.ts
Normal file
@@ -0,0 +1,130 @@
|
||||
/** Health state machine: HEALTHY / DEGRADED / UNAVAILABLE. */
|
||||
|
||||
import { existsSync, readFileSync } from "fs";
|
||||
import { resolve } from "path";
|
||||
import type { ObsidianRagConfig } from "../utils/config.js";
|
||||
|
||||
export type HealthState = "healthy" | "degraded" | "unavailable";
|
||||
|
||||
export interface HealthStatus {
|
||||
state: HealthState;
|
||||
ollama_up: boolean;
|
||||
index_exists: boolean;
|
||||
vault_exists: boolean;
|
||||
total_docs: number;
|
||||
total_chunks: number;
|
||||
last_sync: string | null;
|
||||
active_job: { id: string; mode: string; progress: number } | null;
|
||||
}
|
||||
|
||||
export interface ProbeResult {
|
||||
ollama_up: boolean;
|
||||
index_exists: boolean;
|
||||
vault_exists: boolean;
|
||||
total_docs: number;
|
||||
total_chunks: number;
|
||||
last_sync: string | null;
|
||||
}
|
||||
|
||||
const REPROBE_INTERVAL_MS = 30_000;
|
||||
|
||||
export function createHealthMachine(_config: ObsidianRagConfig) {
|
||||
let currentState: HealthState = "unavailable";
|
||||
let status: ProbeResult = {
|
||||
ollama_up: false,
|
||||
index_exists: false,
|
||||
vault_exists: false,
|
||||
total_docs: 0,
|
||||
total_chunks: 0,
|
||||
last_sync: null,
|
||||
};
|
||||
let activeJob: { id: string; mode: string; progress: number } | null = null;
|
||||
let reprobeTimer: ReturnType<typeof setInterval> | null = null;
|
||||
|
||||
function transition(probe: ProbeResult): void {
|
||||
status = probe;
|
||||
const prev = currentState;
|
||||
if (!probe.index_exists || !probe.vault_exists) {
|
||||
currentState = "unavailable";
|
||||
} else if (!probe.ollama_up) {
|
||||
currentState = "degraded";
|
||||
} else {
|
||||
currentState = "healthy";
|
||||
}
|
||||
if (prev !== currentState) {
|
||||
console.log(`[obsidian-rag] Health: ${prev} → ${currentState}`);
|
||||
}
|
||||
}
|
||||
|
||||
function get(): HealthStatus {
|
||||
return { state: currentState, ...status, active_job: activeJob };
|
||||
}
|
||||
|
||||
function setActiveJob(job: { id: string; mode: string; progress: number } | null): void {
|
||||
activeJob = job;
|
||||
}
|
||||
|
||||
function startReprobing(fn: () => Promise<ProbeResult>): void {
|
||||
if (reprobeTimer) clearInterval(reprobeTimer);
|
||||
reprobeTimer = setInterval(async () => {
|
||||
const probe = await fn();
|
||||
transition(probe);
|
||||
}, REPROBE_INTERVAL_MS);
|
||||
}
|
||||
|
||||
function stop(): void {
|
||||
if (reprobeTimer) {
|
||||
clearInterval(reprobeTimer);
|
||||
reprobeTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
return { get, transition, setActiveJob, startReprobing, stop };
|
||||
}
|
||||
|
||||
export async function probeAll(config: ObsidianRagConfig): Promise<ProbeResult> {
|
||||
const { resolveVectorDbPath } = await import("../utils/lancedb.js");
|
||||
|
||||
const vaultPath = resolve(process.cwd(), config.vault_path);
|
||||
const dbPath = resolveVectorDbPath(config);
|
||||
|
||||
const vaultExists = existsSync(vaultPath);
|
||||
const indexExists = existsSync(String(dbPath));
|
||||
const ollamaUp = await probeOllama(config.embedding.base_url);
|
||||
|
||||
let totalDocs = 0;
|
||||
let totalChunks = 0;
|
||||
let lastSync: string | null = null;
|
||||
|
||||
if (indexExists) {
|
||||
try {
|
||||
const syncPath = resolve(dbPath, "..", "sync-result.json");
|
||||
if (existsSync(syncPath)) {
|
||||
const data = JSON.parse(readFileSync(syncPath, "utf-8"));
|
||||
lastSync = data.timestamp ?? null;
|
||||
totalDocs = data.indexed_files ?? 0;
|
||||
totalChunks = data.total_chunks ?? 0;
|
||||
}
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
ollama_up: ollamaUp,
|
||||
index_exists: indexExists,
|
||||
vault_exists: vaultExists,
|
||||
total_docs: totalDocs,
|
||||
total_chunks: totalChunks,
|
||||
last_sync: lastSync,
|
||||
};
|
||||
}
|
||||
|
||||
async function probeOllama(baseUrl: string): Promise<boolean> {
|
||||
try {
|
||||
const res = await fetch(`${baseUrl}/api/tags`, { signal: AbortSignal.timeout(3000) });
|
||||
return res.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user