feat: refactor curation and vector handling, add new tools for knowledge extraction

curate.js

@@ -1,233 +0,0 @@
-#!/usr/bin/env node
-
-// ─────────────────────────────────────────────────────────────
-//  DEMO: Structured JSON → markdown file
-//  Uses OpenAI API via llm.js
-//  Run:  node curate.js
-// ─────────────────────────────────────────────────────────────
-
-import fs from "fs-extra";
-import path from "path";
-import matter from "gray-matter";
-import readline from "readline";
-import { askOpenAI } from "./llm.js";
-
-// ── CONFIG LOADING ────────────────────────────────────────────
-const DATA_DIR = process.env.DATA_DIR || "/data";
-const CONFIG_PATH = path.join(DATA_DIR, "config.json");
-
-function loadConfig() {
-  try {
-    if (!fs.existsSync(CONFIG_PATH)) {
-      console.error(`Config file not found at ${CONFIG_PATH}`);
-      process.exit(1);
-    }
-    const config = JSON.parse(fs.readFileSync(CONFIG_PATH, "utf8"));
-    return config;
-  } catch (error) {
-    console.error(`Failed to load config from ${CONFIG_PATH}:`, error.message);
-    process.exit(1);
-  }
-}
-
-const config = loadConfig();
-const OUTPUT_DIR = config.OUTPUT_DIR || "/data/context-tree";
-
-// ── STEP 2: Parse the JSON returned ────────────────────
-function parseJSON(rawResponse, fallbackText) {
-  try {
-    // Strip any accidental markdown fences the model added
-    const clean = rawResponse.replace(/```json|```/g, "").trim();
-    const parsed = JSON.parse(clean);
-    console.log("\n✅ JSON parsed successfully");
-    return parsed;
-  } catch (err) {
-    // If the model gave bad JSON, store raw content as a fallback
-    console.log("\n⚠️  JSON parse failed, using fallback structure");
-    return {
-      title: fallbackText.slice(0, 50),
-      topic: "general",
-      type: "context",
-      summary: fallbackText.slice(0, 100),
-      content: fallbackText,
-      facts: [],
-    };
-  }
-}
-
-// ── STEP 3: Make a clean file-safe ID from topic + title ──────
-function makeId(topic, title) {
-  return `${topic}-${title}`
-    .toLowerCase()
-    .replace(/[^a-z0-9]+/g, "-") // replace non-alphanumeric with dash
-    .replace(/^-|-$/g, "") // strip leading/trailing dashes
-    .slice(0, 60); // keep it short
-}
-
-// ── STEP 4: Write the markdown file ───────────────────────────
-async function writeMarkdownFile(entry) {
-  const { id, title, topic, type, content, facts } = entry;
-
-  // Each topic gets its own subfolder
-  const dir = path.join(OUTPUT_DIR, topic);
-  await fs.ensureDir(dir);
-
-  const filePath = path.join(dir, `${id}.md`);
-
-  // Check if this entry already exists (merge if so)
-  let importance = 5;
-  let existingFacts = [];
-
-  if (await fs.pathExists(filePath)) {
-    const existing = matter(await fs.readFile(filePath, "utf-8"));
-    importance = Math.min(10, (existing.data.importance || 5) + 1);
-    existingFacts = existing.data.facts || [];
-    console.log(
-      `\n♻️  Entry exists — merging. Importance boosted to ${importance}`,
-    );
-  }
-
-  // Merge facts, deduplicate
-  const allFacts = [...new Set([...existingFacts, ...facts])];
-
-  const fileContent = matter.stringify(content, {
-    id,
-    title,
-    topic,
-    type,
-    importance,
-    facts: allFacts,
-    updatedAt: new Date().toISOString(),
-  });
-
-  await fs.writeFile(filePath, fileContent, "utf-8");
-  return filePath;
-}
-
-// ── MAIN FLOW ─────────────────────────────────────────────────
-async function curate(rawText) {
-  console.log("\n🚀 Starting curation flow...");
-  console.log(
-    "Input:",
-    rawText.slice(0, 80) + (rawText.length > 80 ? "..." : ""),
-  );
-
-  // Build the extraction prompt
-  const prompt = `You are a knowledge extraction expert. Extract structured knowledge from the content below.
-
-CRITICAL: Return ONLY valid JSON with NO markdown fences, NO explanations, NO extra text. Start with { and end with }.
-
-Content:
-${rawText}
-
-Extract and return EXACTLY this JSON structure:
-{
-  "title": "A clear, specific title (max 10 words). Must capture the essence.",
-  "topic": "SINGLE WORD only: auth, database, api, payments, conventions, infrastructure, testing, or other",
-  "type": "EXACTLY ONE: fact, convention, decision, code, or context",
-  "summary": "One clear sentence explaining the key point.",
-  "content": "Cleaned markdown (use **, -, bullet points). Include all important details from the content.",
-  "facts": ["fact 1 - specific and verifiable", "fact 2 - specific and verifiable", "fact 3 - specific and verifiable"]
-}
-
-RULES:
-- facts array MUST have 2-5 specific, verifiable statements
-- content MUST be cleaned markdown, no raw text
-- title MUST be descriptive and unique
-- Return ONLY the JSON object, nothing else
-- All fields are required`;
-
-  // STEP 1: Get response from OpenAI (via llm.js)
-  const raw = await askOpenAI(rawText, prompt);
-
-  // STEP 2: Parse the JSON
-  const parsed = parseJSON(raw, rawText);
-
-  console.log("\n📦 Parsed structure:");
-  console.log(`  title   : ${parsed.title}`);
-  console.log(`  topic   : ${parsed.topic}`);
-  console.log(`  type    : ${parsed.type}`);
-  console.log(`  summary : ${parsed.summary}`);
-  console.log(`  facts   : ${(parsed.facts || []).length} extracted`);
-  parsed.facts?.forEach((f, i) => console.log(`    [${i + 1}] ${f}`));
-
-  // STEP 3: Make an ID
-  const id = makeId(parsed.topic, parsed.title);
-  console.log(`\n🆔 Generated ID: ${id}`);
-
-  // STEP 4: Write markdown file
-  const filePath = await writeMarkdownFile({
-    id,
-    title: parsed.title,
-    topic: parsed.topic,
-    type: parsed.type,
-    content: parsed.content,
-    facts: parsed.facts || [],
-  });
-
-  console.log(`\n✅ Written to: ${filePath}`);
-
-  // Show what the file looks like
-  const written = await fs.readFile(filePath, "utf-8");
-  console.log("\n📄 File contents:");
-  console.log("─".repeat(50));
-  console.log(written);
-  console.log("─".repeat(50));
-
-  return filePath;
-}
-
-// ── INTERACTIVE CONSOLE INPUT ─────────────────────────────────
-async function main() {
-  console.log("╔══════════════════════════════════════════════╗");
-  console.log("║   JSON → Markdown  |  Curation App         ║");
-  console.log("╚══════════════════════════════════════════════╝");
-  console.log(`Output: ${path.resolve(OUTPUT_DIR)}`);
-
-  // You can also pass text directly as a CLI argument:
-  //   node curate.js "We use PostgreSQL, never MySQL"
-  if (process.argv[2]) {
-    await curate(process.argv[2]);
-    return;
-  }
-
-  // Otherwise interactive mode
-  const rl = readline.createInterface({
-    input: process.stdin,
-    output: process.stdout,
-  });
-
-  console.log('\nType something to curate, or "quit" to exit.');
-  console.log(
-    'Tip: try "We use RS256 for JWT signing, private key in JWT_PRIVATE_KEY env var"\n',
-  );
-
-  const ask = () => {
-    rl.question("📝 Enter text to curate: ", async (input) => {
-      input = input.trim();
-      if (!input || input === "quit") {
-        console.log("Bye!");
-        rl.close();
-        return;
-      }
-
-      try {
-        await curate(input);
-      } catch (err) {
-        if (err.code === "ERR_INVALID_ARG_TYPE") {
-          console.error("\n❌ Missing NVIDIA_API_KEY environment variable");
-          console.error("   Set it: export NVIDIA_API_KEY=your_key");
-        } else {
-          console.error("\n❌ Error:", err.message);
-        }
-      }
-
-      console.log("\n" + "═".repeat(50) + "\n");
-      ask();
-    });
-  };
-
-  ask();
-}
-
-main();

curation.js

@@ -6,55 +6,24 @@ import { askOpenAI } from "./llm.js";
 export async function curateText(rawText, config) {
   const OUTPUT_DIR = config.OUTPUT_DIR || "/data/context-tree";
 
-  console.log("\n🚀 Starting curation flow...");
-  console.log(
-    "Input:",
-    rawText.slice(0, 80) + (rawText.length > 80 ? "..." : ""),
-  );
-
-  // Build extraction prompt
-  const prompt = `You are a knowledge extraction expert. Extract structured knowledge from the content below.
+  const prompt = `Extract knowledge from this text and return ONLY JSON, nothing else.
 
-CRITICAL: Return ONLY valid JSON with NO markdown fences, NO explanations, NO extra text. Start with { and end with }.
-
-Content:
+TEXT:
 ${rawText}
 
-Extract and return EXACTLY this JSON structure:
+RETURN THIS JSON (no markdown, no explanation):
 {
-  "title": "A clear, specific title (max 10 words). Must capture the essence.",
-  "topic": "SINGLE WORD only: auth, database, api, payments, conventions, infrastructure, testing, or other",
-  "type": "EXACTLY ONE: fact, convention, decision, code, or context",
-  "summary": "One clear sentence explaining the key point.",
-  "content": "Cleaned markdown (use **, -, bullet points). Include all important details from the content.",
-  "facts": ["fact 1 - specific and verifiable", "fact 2 - specific and verifiable", "fact 3 - specific and verifiable"]
-}
+  "title": "clear title (max 8 words)",
+  "topic": "ONE WORD: auth OR database OR api OR infrastructure OR OTHER",
+  "type": "ONE: fact OR decision OR context",
+  "summary": "one sentence",
+  "content": "key details as markdown",
+  "facts": ["fact 1", "fact 2"]
+}`;
 
-RULES:
-- facts array MUST have 2-5 specific, verifiable statements
-- content MUST be cleaned markdown, no raw text
-- title MUST be descriptive and unique
-- Return ONLY the JSON object, nothing else
-- All fields are required`;
-
-  // Get response from OpenAI
   const raw = await askOpenAI(rawText, prompt);
-
-  // Parse JSON
   const parsed = parseJSON(raw, rawText);
-
-  console.log("\n📦 Parsed structure:");
-  console.log(`  title   : ${parsed.title}`);
-  console.log(`  topic   : ${parsed.topic}`);
-  console.log(`  type    : ${parsed.type}`);
-  console.log(`  summary : ${parsed.summary}`);
-  console.log(`  facts   : ${(parsed.facts || []).length} extracted`);
-
-  // Generate ID
   const id = makeId(parsed.topic, parsed.title);
-  console.log(`\n🆔 Generated ID: ${id}`);
-
-  // Write markdown file
   const filePath = await writeMarkdownFile(
     {
       id,
@@ -67,8 +36,6 @@ RULES:
     OUTPUT_DIR,
   );
 
-  console.log(`\n✅ Written to: ${filePath}`);
-
   return {
     id,
     title: parsed.title,
@@ -83,19 +50,37 @@ RULES:
 
 function parseJSON(rawResponse, fallbackText) {
   try {
-    const clean = rawResponse.replace(/```json|```/g, "").trim();
+    if (!rawResponse || rawResponse.trim() === "") {
+      throw new Error("Empty response");
+    }
+
+    const clean = rawResponse
+      .replace(/```json/g, "")
+      .replace(/```/g, "")
+      .trim();
+
+    if (!clean.startsWith("{") || !clean.endsWith("}")) {
+      throw new Error("Invalid JSON format");
+    }
+
     const parsed = JSON.parse(clean);
-    console.log("\n✅ JSON parsed successfully");
+
+    if (!parsed.title || !parsed.topic || !parsed.type) {
+      throw new Error("Missing required fields");
+    }
+
     return parsed;
   } catch (err) {
-    console.log("\n⚠️  JSON parse failed, using fallback structure");
+    const lines = fallbackText.split("\n").filter((l) => l.trim());
+    const firstLine = lines[0] || fallbackText;
+
     return {
-      title: fallbackText.slice(0, 50),
+      title: firstLine.slice(0, 50),
       topic: "general",
       type: "context",
-      summary: fallbackText.slice(0, 100),
+      summary: fallbackText.slice(0, 120),
       content: fallbackText,
-      facts: [],
+      facts: [fallbackText.slice(0, 100)],
     };
   }
 }
@@ -123,9 +108,6 @@ async function writeMarkdownFile(entry, outputDir) {
     const existing = matter(await fs.readFile(filePath, "utf-8"));
     importance = Math.min(10, (existing.data.importance || 5) + 1);
     existingFacts = existing.data.facts || [];
-    console.log(
-      `\n♻️  Entry exists — merging. Importance boosted to ${importance}`,
-    );
   }
 
   const allFacts = [...new Set([...existingFacts, ...facts])];

llm.js

@@ -32,33 +32,24 @@ const OPENAI_MODEL = config.OPENAI_MODEL || "stepfun-ai/step-3.5-flash";
 
 // ── STEP 1: Ask OpenAI to extract structured knowledge ────────
 async function askOpenAI(rawText, prompt) {
-  console.log("\n⏳ Sending to OpenAI (NVIDIA)...");
-
   const completion = await openai.chat.completions.create({
     model: OPENAI_MODEL,
     messages: [
       {
         role: "system",
         content:
-          "You are a JSON extraction engine. Return ONLY valid JSON objects with no markdown fences, no explanations, no extra text. Every response must be parseable JSON.",
+          "You are a JSON extraction engine. Return ONLY a valid JSON object. No markdown, no code fences, no text before or after. Your entire response must be parseable JSON.",
       },
       {
         role: "user",
         content: prompt,
       },
     ],
-    temperature: 0.1,
-    top_p: 0.9,
-    max_tokens: 2048,
+    temperature: 0.05,
+    max_tokens: 4096,
   });
 
-  const raw = completion.choices[0]?.message?.content || "";
-  console.log("\n📨 Raw response from OpenAI:");
-  console.log("─".repeat(50));
-  console.log(raw);
-  console.log("─".repeat(50));
-
-  return raw;
+  return completion.choices[0]?.message?.content || "";
 }
 
 export { openai, OPENAI_MODEL, askOpenAI };

server.js

@@ -7,15 +7,8 @@ import {
   ListToolsRequestSchema,
   CallToolRequestSchema,
 } from "@modelcontextprotocol/sdk/types.js";
-import {
-  addDocuments,
-  addDocumentsToolDefinition,
-  listCollectionData,
-  listCollectionDataToolDefinition,
-  queryCollectionData,
-  queryCollectionDataToolDefinition,
-} from "./vector.js";
-import { curateText } from "./curation.js";
+import { vectorTools, handleVectorTool } from "./tools/vectorTool.js";
+import { curateTools, handleCurateTool } from "./tools/curateTool.js";
 
 const PORT = Number(process.env.PORT || 3000);
 const HOST = process.env.HOST || "0.0.0.0";
@@ -52,54 +45,17 @@ function createMcpServer() {
   );
 
   server.setRequestHandler(ListToolsRequestSchema, async () => ({
-    tools: [
-      addDocumentsToolDefinition,
-      listCollectionDataToolDefinition,
-      queryCollectionDataToolDefinition,
-    ],
+    tools: [...curateTools, ...vectorTools],
   }));
 
   server.setRequestHandler(CallToolRequestSchema, async (request) => {
     const { name, arguments: args = {} } = request.params;
 
-    if (name === "list_collection_data") {
-      const results = await listCollectionData(args);
-
-      return {
-        content: [
-          {
-            type: "text",
-            text: JSON.stringify(results, null, 2),
-          },
-        ],
-      };
-    }
-
-    if (name === "add_documents") {
-      const results = await addDocuments(args);
+    const curateResult = await handleCurateTool(name, args, config);
+    if (curateResult) return curateResult;
 
-      return {
-        content: [
-          {
-            type: "text",
-            text: JSON.stringify(results, null, 2),
-          },
-        ],
-      };
-    }
-
-    if (name === "query_collection_data") {
-      const results = await queryCollectionData(args);
-
-      return {
-        content: [
-          {
-            type: "text",
-            text: JSON.stringify(results, null, 2),
-          },
-        ],
-      };
-    }
+    const vectorResult = await handleVectorTool(name, args);
+    if (vectorResult) return vectorResult;
 
     return {
       content: [{ type: "text", text: `Unknown tool: ${name}` }],
@@ -159,32 +115,6 @@ async function main() {
       return;
     }
 
-    if (req.method === "POST" && url.pathname === "/curate") {
-      try {
-        const body = await readBody(req);
-
-        if (!body || typeof body.text !== "string" || !body.text.trim()) {
-          sendJson(res, 400, {
-            error: "Missing or empty 'text' field in request body",
-          });
-          return;
-        }
-
-        const result = await curateText(body.text, config);
-
-        sendJson(res, 200, {
-          success: true,
-          data: result,
-        });
-      } catch (err) {
-        console.error("Curation error:", err);
-        sendJson(res, 500, {
-          error: err.message || "Curation failed",
-        });
-      }
-      return;
-    }
-
     if (req.method === "GET" && url.pathname === "/sse") {
       const transport = new SSEServerTransport("/messages", res);
       transports.set(transport.sessionId, transport);

tools/curateTool.js

@@ -0,0 +1,61 @@
+import { curateText } from "../curation.js";
+
+const curateKnowledgeTool = {
+  name: "curate_knowledge",
+  description:
+    "Extract and curate knowledge from text. Saves structured JSON as markdown file in context-tree.",
+  inputSchema: {
+    type: "object",
+    properties: {
+      text: {
+        type: "string",
+        description: "The text content to curate and extract knowledge from.",
+      },
+    },
+    required: ["text"],
+    additionalProperties: false,
+  },
+};
+
+export const curateTools = [curateKnowledgeTool];
+
+export async function handleCurateTool(name, args, config) {
+  if (name === "curate_knowledge") {
+    try {
+      if (!args.text || typeof args.text !== "string") {
+        return {
+          content: [
+            {
+              type: "text",
+              text: "Error: 'text' field is required and must be a string",
+            },
+          ],
+          isError: true,
+        };
+      }
+
+      const result = await curateText(args.text, config);
+
+      return {
+        content: [
+          {
+            type: "text",
+            text: JSON.stringify(result, null, 2),
+          },
+        ],
+      };
+    } catch (err) {
+      return {
+        content: [
+          {
+            type: "text",
+            text: `Curation error: ${err.message}`,
+          },
+        ],
+        isError: true,
+      };
+    }
+  }
+
+  return null;
+}

tools/vectorTool.js

@@ -0,0 +1,54 @@
+import {
+  addDocuments,
+  addDocumentsToolDefinition,
+  listCollectionData,
+  listCollectionDataToolDefinition,
+  queryCollectionData,
+  queryCollectionDataToolDefinition,
+} from "../vector.js";
+
+export const vectorTools = [
+  addDocumentsToolDefinition,
+  listCollectionDataToolDefinition,
+  queryCollectionDataToolDefinition,
+];
+
+export async function handleVectorTool(name, args) {
+  if (name === "list_collection_data") {
+    const results = await listCollectionData(args);
+    return {
+      content: [
+        {
+          type: "text",
+          text: JSON.stringify(results, null, 2),
+        },
+      ],
+    };
+  }
+
+  if (name === "add_documents") {
+    const results = await addDocuments(args);
+    return {
+      content: [
+        {
+          type: "text",
+          text: JSON.stringify(results, null, 2),
+        },
+      ],
+    };
+  }
+
+  if (name === "query_collection_data") {
+    const results = await queryCollectionData(args);
+    return {
+      content: [
+        {
+          type: "text",
+          text: JSON.stringify(results, null, 2),
+        },
+      ],
+    };
+  }
+
+  return null;
+}