feat: add curation functionality with OpenAI integration

- Implemented `curateText` function in `curation.js` to extract structured knowledge from raw text using OpenAI.
- Added JSON parsing and fallback mechanism for curation results.
- Created markdown file writing logic with importance handling and fact merging.
- Introduced `askOpenAI` function in `llm.js` to interact with OpenAI's API for knowledge extraction.
- Updated server to handle POST requests for curation at the `/curate` endpoint.
- Refactored `server.js` and `vector.js` to use ES module syntax.
- Added necessary dependencies in `package.json` and `package-lock.json`.

Dockerfile

@@ -7,7 +7,7 @@ RUN npm ci --omit=dev
 
 COPY . .
 
-RUN mkdir -p /data && cp config.json /data/config.json
+RUN mkdir -p /data/context-tree && cp config.json /data/config.json
 
 ENV PORT=7860
 ENV DATA_DIR=/data

config.json

@@ -2,5 +2,7 @@
   "CHROMA_URL": "https://maxxcarl-chroma.hf.space",
   "CHROMA_COLLECTION": "mcptest22",
   "EMBEDDER_BASE_URL": "https://maxxcarl-emb1024.hf.space",
-  "EMBEDDER_API_KEY": "Azure123"
+  "EMBEDDER_API_KEY": "Azure123",
+  "OPENAI_MODEL": "stepfun-ai/step-3.5-flash",
+  "OUTPUT_DIR": "/data/context-tree"
 }

curate.js

@@ -0,0 +1,233 @@
+#!/usr/bin/env node
+
+// ─────────────────────────────────────────────────────────────
+//  DEMO: Structured JSON → markdown file
+//  Uses OpenAI API via llm.js
+//  Run:  node curate.js
+// ─────────────────────────────────────────────────────────────
+
+import fs from "fs-extra";
+import path from "path";
+import matter from "gray-matter";
+import readline from "readline";
+import { askOpenAI } from "./llm.js";
+
+// ── CONFIG LOADING ────────────────────────────────────────────
+const DATA_DIR = process.env.DATA_DIR || "/data";
+const CONFIG_PATH = path.join(DATA_DIR, "config.json");
+
+function loadConfig() {
+  try {
+    if (!fs.existsSync(CONFIG_PATH)) {
+      console.error(`Config file not found at ${CONFIG_PATH}`);
+      process.exit(1);
+    }
+    const config = JSON.parse(fs.readFileSync(CONFIG_PATH, "utf8"));
+    return config;
+  } catch (error) {
+    console.error(`Failed to load config from ${CONFIG_PATH}:`, error.message);
+    process.exit(1);
+  }
+}
+
+const config = loadConfig();
+const OUTPUT_DIR = config.OUTPUT_DIR || "/data/context-tree";
+
+// ── STEP 2: Parse the JSON returned ────────────────────
+function parseJSON(rawResponse, fallbackText) {
+  try {
+    // Strip any accidental markdown fences the model added
+    const clean = rawResponse.replace(/```json|```/g, "").trim();
+    const parsed = JSON.parse(clean);
+    console.log("\n✅ JSON parsed successfully");
+    return parsed;
+  } catch (err) {
+    // If the model gave bad JSON, store raw content as a fallback
+    console.log("\n⚠️  JSON parse failed, using fallback structure");
+    return {
+      title: fallbackText.slice(0, 50),
+      topic: "general",
+      type: "context",
+      summary: fallbackText.slice(0, 100),
+      content: fallbackText,
+      facts: [],
+    };
+  }
+}
+
+// ── STEP 3: Make a clean file-safe ID from topic + title ──────
+function makeId(topic, title) {
+  return `${topic}-${title}`
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-") // replace non-alphanumeric with dash
+    .replace(/^-|-$/g, "") // strip leading/trailing dashes
+    .slice(0, 60); // keep it short
+}
+
+// ── STEP 4: Write the markdown file ───────────────────────────
+async function writeMarkdownFile(entry) {
+  const { id, title, topic, type, content, facts } = entry;
+
+  // Each topic gets its own subfolder
+  const dir = path.join(OUTPUT_DIR, topic);
+  await fs.ensureDir(dir);
+
+  const filePath = path.join(dir, `${id}.md`);
+
+  // Check if this entry already exists (merge if so)
+  let importance = 5;
+  let existingFacts = [];
+
+  if (await fs.pathExists(filePath)) {
+    const existing = matter(await fs.readFile(filePath, "utf-8"));
+    importance = Math.min(10, (existing.data.importance || 5) + 1);
+    existingFacts = existing.data.facts || [];
+    console.log(
+      `\n♻️  Entry exists — merging. Importance boosted to ${importance}`,
+    );
+  }
+
+  // Merge facts, deduplicate
+  const allFacts = [...new Set([...existingFacts, ...facts])];
+
+  const fileContent = matter.stringify(content, {
+    id,
+    title,
+    topic,
+    type,
+    importance,
+    facts: allFacts,
+    updatedAt: new Date().toISOString(),
+  });
+
+  await fs.writeFile(filePath, fileContent, "utf-8");
+  return filePath;
+}
+
+// ── MAIN FLOW ─────────────────────────────────────────────────
+async function curate(rawText) {
+  console.log("\n🚀 Starting curation flow...");
+  console.log(
+    "Input:",
+    rawText.slice(0, 80) + (rawText.length > 80 ? "..." : ""),
+  );
+
+  // Build the extraction prompt
+  const prompt = `You are a knowledge extraction expert. Extract structured knowledge from the content below.
+
+CRITICAL: Return ONLY valid JSON with NO markdown fences, NO explanations, NO extra text. Start with { and end with }.
+
+Content:
+${rawText}
+
+Extract and return EXACTLY this JSON structure:
+{
+  "title": "A clear, specific title (max 10 words). Must capture the essence.",
+  "topic": "SINGLE WORD only: auth, database, api, payments, conventions, infrastructure, testing, or other",
+  "type": "EXACTLY ONE: fact, convention, decision, code, or context",
+  "summary": "One clear sentence explaining the key point.",
+  "content": "Cleaned markdown (use **, -, bullet points). Include all important details from the content.",
+  "facts": ["fact 1 - specific and verifiable", "fact 2 - specific and verifiable", "fact 3 - specific and verifiable"]
+}
+
+RULES:
+- facts array MUST have 2-5 specific, verifiable statements
+- content MUST be cleaned markdown, no raw text
+- title MUST be descriptive and unique
+- Return ONLY the JSON object, nothing else
+- All fields are required`;
+
+  // STEP 1: Get response from OpenAI (via llm.js)
+  const raw = await askOpenAI(rawText, prompt);
+
+  // STEP 2: Parse the JSON
+  const parsed = parseJSON(raw, rawText);
+
+  console.log("\n📦 Parsed structure:");
+  console.log(`  title   : ${parsed.title}`);
+  console.log(`  topic   : ${parsed.topic}`);
+  console.log(`  type    : ${parsed.type}`);
+  console.log(`  summary : ${parsed.summary}`);
+  console.log(`  facts   : ${(parsed.facts || []).length} extracted`);
+  parsed.facts?.forEach((f, i) => console.log(`    [${i + 1}] ${f}`));
+
+  // STEP 3: Make an ID
+  const id = makeId(parsed.topic, parsed.title);
+  console.log(`\n🆔 Generated ID: ${id}`);
+
+  // STEP 4: Write markdown file
+  const filePath = await writeMarkdownFile({
+    id,
+    title: parsed.title,
+    topic: parsed.topic,
+    type: parsed.type,
+    content: parsed.content,
+    facts: parsed.facts || [],
+  });
+
+  console.log(`\n✅ Written to: ${filePath}`);
+
+  // Show what the file looks like
+  const written = await fs.readFile(filePath, "utf-8");
+  console.log("\n📄 File contents:");
+  console.log("─".repeat(50));
+  console.log(written);
+  console.log("─".repeat(50));
+
+  return filePath;
+}
+
+// ── INTERACTIVE CONSOLE INPUT ─────────────────────────────────
+async function main() {
+  console.log("╔══════════════════════════════════════════════╗");
+  console.log("║   JSON → Markdown  |  Curation App         ║");
+  console.log("╚══════════════════════════════════════════════╝");
+  console.log(`Output: ${path.resolve(OUTPUT_DIR)}`);
+
+  // You can also pass text directly as a CLI argument:
+  //   node curate.js "We use PostgreSQL, never MySQL"
+  if (process.argv[2]) {
+    await curate(process.argv[2]);
+    return;
+  }
+
+  // Otherwise interactive mode
+  const rl = readline.createInterface({
+    input: process.stdin,
+    output: process.stdout,
+  });
+
+  console.log('\nType something to curate, or "quit" to exit.');
+  console.log(
+    'Tip: try "We use RS256 for JWT signing, private key in JWT_PRIVATE_KEY env var"\n',
+  );
+
+  const ask = () => {
+    rl.question("📝 Enter text to curate: ", async (input) => {
+      input = input.trim();
+      if (!input || input === "quit") {
+        console.log("Bye!");
+        rl.close();
+        return;
+      }
+
+      try {
+        await curate(input);
+      } catch (err) {
+        if (err.code === "ERR_INVALID_ARG_TYPE") {
+          console.error("\n❌ Missing NVIDIA_API_KEY environment variable");
+          console.error("   Set it: export NVIDIA_API_KEY=your_key");
+        } else {
+          console.error("\n❌ Error:", err.message);
+        }
+      }
+
+      console.log("\n" + "═".repeat(50) + "\n");
+      ask();
+    });
+  };
+
+  ask();
+}
+
+main();

curation.js

@@ -0,0 +1,145 @@
+import fs from "fs-extra";
+import path from "path";
+import matter from "gray-matter";
+import { askOpenAI } from "./llm.js";
+
+export async function curateText(rawText, config) {
+  const OUTPUT_DIR = config.OUTPUT_DIR || "/data/context-tree";
+
+  console.log("\n🚀 Starting curation flow...");
+  console.log(
+    "Input:",
+    rawText.slice(0, 80) + (rawText.length > 80 ? "..." : ""),
+  );
+
+  // Build extraction prompt
+  const prompt = `You are a knowledge extraction expert. Extract structured knowledge from the content below.
+
+CRITICAL: Return ONLY valid JSON with NO markdown fences, NO explanations, NO extra text. Start with { and end with }.
+
+Content:
+${rawText}
+
+Extract and return EXACTLY this JSON structure:
+{
+  "title": "A clear, specific title (max 10 words). Must capture the essence.",
+  "topic": "SINGLE WORD only: auth, database, api, payments, conventions, infrastructure, testing, or other",
+  "type": "EXACTLY ONE: fact, convention, decision, code, or context",
+  "summary": "One clear sentence explaining the key point.",
+  "content": "Cleaned markdown (use **, -, bullet points). Include all important details from the content.",
+  "facts": ["fact 1 - specific and verifiable", "fact 2 - specific and verifiable", "fact 3 - specific and verifiable"]
+}
+
+RULES:
+- facts array MUST have 2-5 specific, verifiable statements
+- content MUST be cleaned markdown, no raw text
+- title MUST be descriptive and unique
+- Return ONLY the JSON object, nothing else
+- All fields are required`;
+
+  // Get response from OpenAI
+  const raw = await askOpenAI(rawText, prompt);
+
+  // Parse JSON
+  const parsed = parseJSON(raw, rawText);
+
+  console.log("\n📦 Parsed structure:");
+  console.log(`  title   : ${parsed.title}`);
+  console.log(`  topic   : ${parsed.topic}`);
+  console.log(`  type    : ${parsed.type}`);
+  console.log(`  summary : ${parsed.summary}`);
+  console.log(`  facts   : ${(parsed.facts || []).length} extracted`);
+
+  // Generate ID
+  const id = makeId(parsed.topic, parsed.title);
+  console.log(`\n🆔 Generated ID: ${id}`);
+
+  // Write markdown file
+  const filePath = await writeMarkdownFile(
+    {
+      id,
+      title: parsed.title,
+      topic: parsed.topic,
+      type: parsed.type,
+      content: parsed.content,
+      facts: parsed.facts || [],
+    },
+    OUTPUT_DIR,
+  );
+
+  console.log(`\n✅ Written to: ${filePath}`);
+
+  return {
+    id,
+    title: parsed.title,
+    topic: parsed.topic,
+    type: parsed.type,
+    summary: parsed.summary,
+    content: parsed.content,
+    facts: parsed.facts || [],
+    filePath,
+  };
+}
+
+function parseJSON(rawResponse, fallbackText) {
+  try {
+    const clean = rawResponse.replace(/```json|```/g, "").trim();
+    const parsed = JSON.parse(clean);
+    console.log("\n✅ JSON parsed successfully");
+    return parsed;
+  } catch (err) {
+    console.log("\n⚠️  JSON parse failed, using fallback structure");
+    return {
+      title: fallbackText.slice(0, 50),
+      topic: "general",
+      type: "context",
+      summary: fallbackText.slice(0, 100),
+      content: fallbackText,
+      facts: [],
+    };
+  }
+}
+
+function makeId(topic, title) {
+  return `${topic}-${title}`
+    .toLowerCase()
+    .replace(/[^a-z0-9]+/g, "-")
+    .replace(/^-|-$/g, "")
+    .slice(0, 60);
+}
+
+async function writeMarkdownFile(entry, outputDir) {
+  const { id, title, topic, type, content, facts } = entry;
+
+  const dir = path.join(outputDir, topic);
+  await fs.ensureDir(dir);
+
+  const filePath = path.join(dir, `${id}.md`);
+
+  let importance = 5;
+  let existingFacts = [];
+
+  if (await fs.pathExists(filePath)) {
+    const existing = matter(await fs.readFile(filePath, "utf-8"));
+    importance = Math.min(10, (existing.data.importance || 5) + 1);
+    existingFacts = existing.data.facts || [];
+    console.log(
+      `\n♻️  Entry exists — merging. Importance boosted to ${importance}`,
+    );
+  }
+
+  const allFacts = [...new Set([...existingFacts, ...facts])];
+
+  const fileContent = matter.stringify(content, {
+    id,
+    title,
+    topic,
+    type,
+    importance,
+    facts: allFacts,
+    updatedAt: new Date().toISOString(),
+  });
+
+  await fs.writeFile(filePath, fileContent, "utf-8");
+  return filePath;
+}

llm.js

@@ -0,0 +1,64 @@
+import OpenAI from "openai";
+import fs from "node:fs";
+import path from "node:path";
+
+// ── CONFIG LOADING ────────────────────────────────────────────
+const DATA_DIR = process.env.DATA_DIR || "/data";
+const CONFIG_PATH = path.join(DATA_DIR, "config.json");
+
+function loadConfig() {
+  try {
+    if (!fs.existsSync(CONFIG_PATH)) {
+      console.error(`Config file not found at ${CONFIG_PATH}`);
+      process.exit(1);
+    }
+    const config = JSON.parse(fs.readFileSync(CONFIG_PATH, "utf8"));
+    return config;
+  } catch (error) {
+    console.error(`Failed to load config from ${CONFIG_PATH}:`, error.message);
+    process.exit(1);
+  }
+}
+
+const config = loadConfig();
+
+// ── OPENAI CONFIG ────────────────────────────────────────────
+const openai = new OpenAI({
+  apiKey: process.env.NVIDIA_API_KEY,
+  baseURL: "https://integrate.api.nvidia.com/v1",
+});
+
+const OPENAI_MODEL = config.OPENAI_MODEL || "stepfun-ai/step-3.5-flash";
+
+// ── STEP 1: Ask OpenAI to extract structured knowledge ────────
+async function askOpenAI(rawText, prompt) {
+  console.log("\n⏳ Sending to OpenAI (NVIDIA)...");
+
+  const completion = await openai.chat.completions.create({
+    model: OPENAI_MODEL,
+    messages: [
+      {
+        role: "system",
+        content:
+          "You are a JSON extraction engine. Return ONLY valid JSON objects with no markdown fences, no explanations, no extra text. Every response must be parseable JSON.",
+      },
+      {
+        role: "user",
+        content: prompt,
+      },
+    ],
+    temperature: 0.1,
+    top_p: 0.9,
+    max_tokens: 2048,
+  });
+
+  const raw = completion.choices[0]?.message?.content || "";
+  console.log("\n📨 Raw response from OpenAI:");
+  console.log("─".repeat(50));
+  console.log(raw);
+  console.log("─".repeat(50));
+
+  return raw;
+}
+
+export { openai, OPENAI_MODEL, askOpenAI };

package-lock.json

@@ -10,9 +10,15 @@
       "license": "ISC",
       "dependencies": {
         "@modelcontextprotocol/sdk": "^1.29.0",
+        "axios": "^1.16.1",
         "chroma": "^0.0.1",
         "chromadb": "^3.4.3",
-        "express": "^5.2.1"
+        "express": "^5.2.1",
+        "fs-extra": "^11.3.5",
+        "gray-matter": "^4.0.3",
+        "openai": "^6.39.0",
+        "path": "^0.12.7",
+        "readline": "^1.3.0"
       }
     },
     "node_modules/@hono/node-server": {
@@ -80,6 +86,18 @@
         "node": ">= 0.6"
       }
     },
+    "node_modules/agent-base": {
+      "version": "6.0.2",
+      "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-6.0.2.tgz",
+      "integrity": "sha512-RZNwNclF7+MS/8bDg70amg32dyeZGZxiDuQmZxKLAlQjr3jGyLx+4Kkk58UO7D2QdgFIQCovuSuZESne6RG6XQ==",
+      "license": "MIT",
+      "dependencies": {
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 6.0.0"
+      }
+    },
     "node_modules/ajv": {
       "version": "8.20.0",
       "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz",
@@ -113,6 +131,33 @@
         }
       }
     },
+    "node_modules/argparse": {
+      "version": "1.0.10",
+      "resolved": "https://registry.npmjs.org/argparse/-/argparse-1.0.10.tgz",
+      "integrity": "sha512-o5Roy6tNG4SL/FOkCAN6RzjiakZS25RLYFrcMttJqbdd8BWrnA+fGz57iN5Pb06pvBGvl5gQ0B48dJlslXvoTg==",
+      "license": "MIT",
+      "dependencies": {
+        "sprintf-js": "~1.0.2"
+      }
+    },
+    "node_modules/asynckit": {
+      "version": "0.4.0",
+      "resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
+      "integrity": "sha512-Oei9OH4tRh0YqU3GxhX79dM/mwVgvbZJaSNaRk+bshkj0S5cfHcgYakreBjrHwatXKbz+IoIdYLxrKim2MjW0Q==",
+      "license": "MIT"
+    },
+    "node_modules/axios": {
+      "version": "1.16.1",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-1.16.1.tgz",
+      "integrity": "sha512-caYkukvroVPO8KrzuJEb50Hm07KwfBZPEC3VeFHTsqWHvKTsy54hjJz9BS/cdaypROE2rH6xvm9mHX4fgWkr3A==",
+      "license": "MIT",
+      "dependencies": {
+        "follow-redirects": "^1.16.0",
+        "form-data": "^4.0.5",
+        "https-proxy-agent": "^5.0.1",
+        "proxy-from-env": "^2.1.0"
+      }
+    },
     "node_modules/body-parser": {
       "version": "2.2.2",
       "resolved": "https://registry.npmjs.org/body-parser/-/body-parser-2.2.2.tgz",
@@ -288,6 +333,18 @@
         "node": ">= 10"
       }
     },
+    "node_modules/combined-stream": {
+      "version": "1.0.8",
+      "resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.8.tgz",
+      "integrity": "sha512-FQN4MRfuJeHf7cBbBMJFXhKSDq+2kAArBlmRBvcvFE5BB1HZKXtSFASDhdlz9zOYwxh8lDdnvmMOe/+5cdoEdg==",
+      "license": "MIT",
+      "dependencies": {
+        "delayed-stream": "~1.0.0"
+      },
+      "engines": {
+        "node": ">= 0.8"
+      }
+    },
     "node_modules/content-disposition": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-1.1.0.tgz",
@@ -376,6 +433,15 @@
         }
       }
     },
+    "node_modules/delayed-stream": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
+      "integrity": "sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.4.0"
+      }
+    },
     "node_modules/depd": {
       "version": "2.0.0",
       "resolved": "https://registry.npmjs.org/depd/-/depd-2.0.0.tgz",
@@ -444,12 +510,40 @@
         "node": ">= 0.4"
       }
     },
+    "node_modules/es-set-tostringtag": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/es-set-tostringtag/-/es-set-tostringtag-2.1.0.tgz",
+      "integrity": "sha512-j6vWzfrGVfyXxge+O0x5sh6cvxAog0a/4Rdd2K36zCMV5eJ+/+tOAngRO8cODMNWbVRdVlmGZQL2YS3yR8bIUA==",
+      "license": "MIT",
+      "dependencies": {
+        "es-errors": "^1.3.0",
+        "get-intrinsic": "^1.2.6",
+        "has-tostringtag": "^1.0.2",
+        "hasown": "^2.0.2"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      }
+    },
     "node_modules/escape-html": {
       "version": "1.0.3",
       "resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
       "integrity": "sha512-NiSupZ4OeuGwr68lGIeym/ksIZMJodUGOSCZ/FSnTxcrekbvqrgdUxlJOMpijaKZVjAJrWrGs/6Jy8OMuyj9ow==",
       "license": "MIT"
     },
+    "node_modules/esprima": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/esprima/-/esprima-4.0.1.tgz",
+      "integrity": "sha512-eGuFFw7Upda+g4p+QHvnW0RyTX/SVeJBDM/gCtMARO0cLuT2HcEKnTPvhjV6aGeqrCB/sbNop0Kszm0jsaWU4A==",
+      "license": "BSD-2-Clause",
+      "bin": {
+        "esparse": "bin/esparse.js",
+        "esvalidate": "bin/esvalidate.js"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/etag": {
       "version": "1.8.1",
       "resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
@@ -541,6 +635,18 @@
         "express": ">= 4.11"
       }
     },
+    "node_modules/extend-shallow": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/extend-shallow/-/extend-shallow-2.0.1.tgz",
+      "integrity": "sha512-zCnTtlxNoAiDc3gqY2aYAWFx7XWWiasuF2K8Me5WbN8otHKTUKBwjPtNpRs/rbUZm7KxWAaNj7P1a/p52GbVug==",
+      "license": "MIT",
+      "dependencies": {
+        "is-extendable": "^0.1.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/fast-deep-equal": {
       "version": "3.1.3",
       "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz",
@@ -584,6 +690,63 @@
         "url": "https://opencollective.com/express"
       }
     },
+    "node_modules/follow-redirects": {
+      "version": "1.16.0",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.16.0.tgz",
+      "integrity": "sha512-y5rN/uOsadFT/JfYwhxRS5R7Qce+g3zG97+JrtFZlC9klX/W5hD7iiLzScI4nZqUS7DNUdhPgw4xI8W2LuXlUw==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "license": "MIT",
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
+    "node_modules/form-data": {
+      "version": "4.0.5",
+      "resolved": "https://registry.npmjs.org/form-data/-/form-data-4.0.5.tgz",
+      "integrity": "sha512-8RipRLol37bNs2bhoV67fiTEvdTrbMUYcFTiy3+wuuOnUog2QBHCZWXDRijWQfAkhBj2Uf5UnVaiWwA5vdd82w==",
+      "license": "MIT",
+      "dependencies": {
+        "asynckit": "^0.4.0",
+        "combined-stream": "^1.0.8",
+        "es-set-tostringtag": "^2.1.0",
+        "hasown": "^2.0.2",
+        "mime-types": "^2.1.12"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
+    "node_modules/form-data/node_modules/mime-db": {
+      "version": "1.52.0",
+      "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.52.0.tgz",
+      "integrity": "sha512-sPU4uV7dYlvtWJxwwxHD0PuihVNiE7TyAbQ5SWxDCB9mUYvOgroQOwYQQOKPJ8CIbE+1ETVlOoK1UC2nU3gYvg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
+    "node_modules/form-data/node_modules/mime-types": {
+      "version": "2.1.35",
+      "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.35.tgz",
+      "integrity": "sha512-ZDY+bPm5zTTF+YpCrAU9nK0UgICYPT0QtT1NZWFv4s++TNkcgVaT0g6+4R2uI4MjQjzysHB1zxuWL50hzaeXiw==",
+      "license": "MIT",
+      "dependencies": {
+        "mime-db": "1.52.0"
+      },
+      "engines": {
+        "node": ">= 0.6"
+      }
+    },
     "node_modules/forwarded": {
       "version": "0.2.0",
       "resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.2.0.tgz",
@@ -602,6 +765,20 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/fs-extra": {
+      "version": "11.3.5",
+      "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-11.3.5.tgz",
+      "integrity": "sha512-eKpRKAovdpZtR1WopLHxlBWvAgPny3c4gX1G5Jhwmmw4XJj0ifSD5qB5TOo8hmA0wlRKDAOAhEE1yVPgs6Fgcg==",
+      "license": "MIT",
+      "dependencies": {
+        "graceful-fs": "^4.2.0",
+        "jsonfile": "^6.0.1",
+        "universalify": "^2.0.0"
+      },
+      "engines": {
+        "node": ">=14.14"
+      }
+    },
     "node_modules/function-bind": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz",
@@ -660,6 +837,27 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/graceful-fs": {
+      "version": "4.2.11",
+      "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.11.tgz",
+      "integrity": "sha512-RbJ5/jmFcNNCcDV5o9eTnBLJ/HszWV0P73bc+Ff4nS/rJj+YaS6IGyiOL0VoBYX+l1Wrl3k63h/KrH+nhJ0XvQ==",
+      "license": "ISC"
+    },
+    "node_modules/gray-matter": {
+      "version": "4.0.3",
+      "resolved": "https://registry.npmjs.org/gray-matter/-/gray-matter-4.0.3.tgz",
+      "integrity": "sha512-5v6yZd4JK3eMI3FqqCouswVqwugaA9r4dNZB1wwcmrD02QkV5H0y7XBQW8QwQqEaZY1pM9aqORSORhJRdNK44Q==",
+      "license": "MIT",
+      "dependencies": {
+        "js-yaml": "^3.13.1",
+        "kind-of": "^6.0.2",
+        "section-matter": "^1.0.0",
+        "strip-bom-string": "^1.0.0"
+      },
+      "engines": {
+        "node": ">=6.0"
+      }
+    },
     "node_modules/has-symbols": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.1.0.tgz",
@@ -672,6 +870,21 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/has-tostringtag": {
+      "version": "1.0.2",
+      "resolved": "https://registry.npmjs.org/has-tostringtag/-/has-tostringtag-1.0.2.tgz",
+      "integrity": "sha512-NqADB8VjPFLM2V0VvHUewwwsw0ZWBaIdgo+ieHtK3hasLz4qeCRjYcqfB6AQrBggRKppKF8L52/VqdVsO47Dlw==",
+      "license": "MIT",
+      "dependencies": {
+        "has-symbols": "^1.0.3"
+      },
+      "engines": {
+        "node": ">= 0.4"
+      },
+      "funding": {
+        "url": "https://github.com/sponsors/ljharb"
+      }
+    },
     "node_modules/hasown": {
       "version": "2.0.3",
       "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.3.tgz",
@@ -713,6 +926,19 @@
         "url": "https://opencollective.com/express"
       }
     },
+    "node_modules/https-proxy-agent": {
+      "version": "5.0.1",
+      "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-5.0.1.tgz",
+      "integrity": "sha512-dFcAjpTQFgoLMzC2VwU+C/CbS7uRL0lWmxDITmqm7C+7F0Odmj6s9l6alZc6AELXhrnggM2CeWSXHGOdX2YtwA==",
+      "license": "MIT",
+      "dependencies": {
+        "agent-base": "6",
+        "debug": "4"
+      },
+      "engines": {
+        "node": ">= 6"
+      }
+    },
     "node_modules/iconv-lite": {
       "version": "0.7.2",
       "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.7.2.tgz",
@@ -753,6 +979,15 @@
         "node": ">= 0.10"
       }
     },
+    "node_modules/is-extendable": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/is-extendable/-/is-extendable-0.1.1.tgz",
+      "integrity": "sha512-5BMULNob1vgFX6EjQw5izWDxrecWK9AM72rugNr0TFldMOi0fj6Jk+zeKIt0xGj4cEfQIJth4w3OKWOJ4f+AFw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/is-promise": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/is-promise/-/is-promise-4.0.0.tgz",
@@ -774,6 +1009,19 @@
         "url": "https://github.com/sponsors/panva"
       }
     },
+    "node_modules/js-yaml": {
+      "version": "3.14.2",
+      "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-3.14.2.tgz",
+      "integrity": "sha512-PMSmkqxr106Xa156c2M265Z+FTrPl+oxd/rgOQy2tijQeK5TxQ43psO1ZCwhVOSdnn+RzkzlRz/eY4BgJBYVpg==",
+      "license": "MIT",
+      "dependencies": {
+        "argparse": "^1.0.7",
+        "esprima": "^4.0.0"
+      },
+      "bin": {
+        "js-yaml": "bin/js-yaml.js"
+      }
+    },
     "node_modules/json-schema-traverse": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz",
@@ -786,6 +1034,27 @@
       "integrity": "sha512-fQhoXdcvc3V28x7C7BMs4P5+kNlgUURe2jmUT1T//oBRMDrqy1QPelJimwZGo7Hg9VPV3EQV5Bnq4hbFy2vetA==",
       "license": "BSD-2-Clause"
     },
+    "node_modules/jsonfile": {
+      "version": "6.2.1",
+      "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.2.1.tgz",
+      "integrity": "sha512-zwOTdL3rFQ/lRdBnntKVOX6k5cKJwEc1HdilT71BWEu7J41gXIB2MRp+vxduPSwZJPWBxEzv4yH1wYLJGUHX4Q==",
+      "license": "MIT",
+      "dependencies": {
+        "universalify": "^2.0.0"
+      },
+      "optionalDependencies": {
+        "graceful-fs": "^4.1.6"
+      }
+    },
+    "node_modules/kind-of": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
+      "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/math-intrinsics": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/math-intrinsics/-/math-intrinsics-1.1.0.tgz",
@@ -898,6 +1167,27 @@
         "wrappy": "1"
       }
     },
+    "node_modules/openai": {
+      "version": "6.39.0",
+      "resolved": "https://registry.npmjs.org/openai/-/openai-6.39.0.tgz",
+      "integrity": "sha512-O61LIsimY3acVabwvomwFhwrnN36yvHY2quIfy9keEcFytGgWeV35yLHQ6NVMLSBxRpHmcg2yuhCnlu2HT4pLQ==",
+      "license": "Apache-2.0",
+      "bin": {
+        "openai": "bin/cli"
+      },
+      "peerDependencies": {
+        "ws": "^8.18.0",
+        "zod": "^3.25 || ^4.0"
+      },
+      "peerDependenciesMeta": {
+        "ws": {
+          "optional": true
+        },
+        "zod": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/parseurl": {
       "version": "1.3.3",
       "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz",
@@ -907,6 +1197,16 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/path": {
+      "version": "0.12.7",
+      "resolved": "https://registry.npmjs.org/path/-/path-0.12.7.tgz",
+      "integrity": "sha512-aXXC6s+1w7otVF9UletFkFcDsJeO7lSZBPUQhtb5O0xJe8LtYhj/GxldoL09bBj9+ZmE2hNoHqQSFMN5fikh4Q==",
+      "license": "MIT",
+      "dependencies": {
+        "process": "^0.11.1",
+        "util": "^0.10.3"
+      }
+    },
     "node_modules/path-key": {
       "version": "3.1.1",
       "resolved": "https://registry.npmjs.org/path-key/-/path-key-3.1.1.tgz",
@@ -935,6 +1235,15 @@
         "node": ">=16.20.0"
       }
     },
+    "node_modules/process": {
+      "version": "0.11.10",
+      "resolved": "https://registry.npmjs.org/process/-/process-0.11.10.tgz",
+      "integrity": "sha512-cdGef/drWFoydD1JsMzuFf8100nZl+GT+yacc2bEced5f9Rjk4z+WtFUTBu9PhOi9j/jfmBPu0mMEY4wIdAF8A==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 0.6.0"
+      }
+    },
     "node_modules/proxy-addr": {
       "version": "2.0.7",
       "resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.7.tgz",
@@ -948,6 +1257,15 @@
         "node": ">= 0.10"
       }
     },
+    "node_modules/proxy-from-env": {
+      "version": "2.1.0",
+      "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz",
+      "integrity": "sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=10"
+      }
+    },
     "node_modules/qs": {
       "version": "6.15.2",
       "resolved": "https://registry.npmjs.org/qs/-/qs-6.15.2.tgz",
@@ -987,6 +1305,12 @@
         "node": ">= 0.10"
       }
     },
+    "node_modules/readline": {
+      "version": "1.3.0",
+      "resolved": "https://registry.npmjs.org/readline/-/readline-1.3.0.tgz",
+      "integrity": "sha512-k2d6ACCkiNYz222Fs/iNze30rRJ1iIicW7JuX/7/cozvih6YCkFZH+J6mAFDVgv0dRBaAyr4jDqC95R2y4IADg==",
+      "license": "BSD"
+    },
     "node_modules/require-from-string": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz",
@@ -1018,6 +1342,19 @@
       "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==",
       "license": "MIT"
     },
+    "node_modules/section-matter": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/section-matter/-/section-matter-1.0.0.tgz",
+      "integrity": "sha512-vfD3pmTzGpufjScBh50YHKzEu2lxBWhVEHsNGoEXmCmn2hKGfeNLYMzCJpe8cD7gqX7TJluOVpBkAequ6dgMmA==",
+      "license": "MIT",
+      "dependencies": {
+        "extend-shallow": "^2.0.1",
+        "kind-of": "^6.0.0"
+      },
+      "engines": {
+        "node": ">=4"
+      }
+    },
     "node_modules/semver": {
       "version": "7.8.1",
       "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.1.tgz",
@@ -1174,6 +1511,12 @@
         "url": "https://github.com/sponsors/ljharb"
       }
     },
+    "node_modules/sprintf-js": {
+      "version": "1.0.3",
+      "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.0.3.tgz",
+      "integrity": "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g==",
+      "license": "BSD-3-Clause"
+    },
     "node_modules/statuses": {
       "version": "2.0.2",
       "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.2.tgz",
@@ -1183,6 +1526,15 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/strip-bom-string": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/strip-bom-string/-/strip-bom-string-1.0.0.tgz",
+      "integrity": "sha512-uCC2VHvQRYu+lMh4My/sFNmF2klFymLX1wHJeXnbEJERpV/ZsVuonzerjfrGpIGF7LBVa1O7i9kjiWvJiFck8g==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
     "node_modules/toidentifier": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.1.tgz",
@@ -1223,6 +1575,15 @@
         "url": "https://opencollective.com/express"
       }
     },
+    "node_modules/universalify": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.1.tgz",
+      "integrity": "sha512-gptHNQghINnc/vTGIk0SOFGFNXw7JVrlRUtConJRlvaw6DuX0wO5Jeko9sWrMBhh+PsYAZ7oXAiOnf/UKogyiw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">= 10.0.0"
+      }
+    },
     "node_modules/unpipe": {
       "version": "1.0.0",
       "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
@@ -1232,6 +1593,21 @@
         "node": ">= 0.8"
       }
     },
+    "node_modules/util": {
+      "version": "0.10.4",
+      "resolved": "https://registry.npmjs.org/util/-/util-0.10.4.tgz",
+      "integrity": "sha512-0Pm9hTQ3se5ll1XihRic3FDIku70C+iHUdT/W926rSgHV5QgXsYbKZN8MSC3tJtSkhuROzvsQjAaFENRXr+19A==",
+      "license": "MIT",
+      "dependencies": {
+        "inherits": "2.0.3"
+      }
+    },
+    "node_modules/util/node_modules/inherits": {
+      "version": "2.0.3",
+      "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
+      "integrity": "sha512-x00IRNXNy63jwGkJmzPigoySHbaqpNuzKbBOmzK+g2OdZpQ9w+sxCN+VSB3ja7IAge2OP2qpfxTjeNcyjmW1uw==",
+      "license": "ISC"
+    },
     "node_modules/vary": {
       "version": "1.1.2",
       "resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",

package.json

@@ -14,11 +14,17 @@
   "keywords": [],
   "author": "",
   "license": "ISC",
-  "type": "commonjs",
+  "type": "module",
   "dependencies": {
     "@modelcontextprotocol/sdk": "^1.29.0",
+    "axios": "^1.16.1",
     "chroma": "^0.0.1",
     "chromadb": "^3.4.3",
-    "express": "^5.2.1"
+    "express": "^5.2.1",
+    "fs-extra": "^11.3.5",
+    "gray-matter": "^4.0.3",
+    "openai": "^6.39.0",
+    "path": "^0.12.7",
+    "readline": "^1.3.0"
   }
 }

server.js

@@ -1,22 +1,21 @@
-const http = require("node:http");
-const fs = require("node:fs");
-const path = require("node:path");
-const { Server } = require("@modelcontextprotocol/sdk/server/index.js");
-const {
-  SSEServerTransport,
-} = require("@modelcontextprotocol/sdk/server/sse.js");
-const {
+import http from "node:http";
+import fs from "node:fs";
+import path from "node:path";
+import { Server } from "@modelcontextprotocol/sdk/server/index.js";
+import { SSEServerTransport } from "@modelcontextprotocol/sdk/server/sse.js";
+import {
   ListToolsRequestSchema,
   CallToolRequestSchema,
-} = require("@modelcontextprotocol/sdk/types.js");
-const {
+} from "@modelcontextprotocol/sdk/types.js";
+import {
   addDocuments,
   addDocumentsToolDefinition,
   listCollectionData,
   listCollectionDataToolDefinition,
   queryCollectionData,
   queryCollectionDataToolDefinition,
-} = require("./vector");
+} from "./vector.js";
+import { curateText } from "./curation.js";
 
 const PORT = Number(process.env.PORT || 3000);
 const HOST = process.env.HOST || "0.0.0.0";
@@ -160,6 +159,32 @@ async function main() {
       return;
     }
 
+    if (req.method === "POST" && url.pathname === "/curate") {
+      try {
+        const body = await readBody(req);
+
+        if (!body || typeof body.text !== "string" || !body.text.trim()) {
+          sendJson(res, 400, {
+            error: "Missing or empty 'text' field in request body",
+          });
+          return;
+        }
+
+        const result = await curateText(body.text, config);
+
+        sendJson(res, 200, {
+          success: true,
+          data: result,
+        });
+      } catch (err) {
+        console.error("Curation error:", err);
+        sendJson(res, 500, {
+          error: err.message || "Curation failed",
+        });
+      }
+      return;
+    }
+
     if (req.method === "GET" && url.pathname === "/sse") {
       const transport = new SSEServerTransport("/messages", res);
       transports.set(transport.sessionId, transport);

vector.js

@@ -1,5 +1,8 @@
-const fs = require("node:fs");
-const path = require("node:path");
+import fs from "node:fs";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
 
 async function fetchJson(url, options = {}) {
   const response = await fetch(url, options);
@@ -27,7 +30,8 @@ async function fetchJson(url, options = {}) {
 }
 
 function loadConfig() {
-  const configPath = path.join(__dirname, "config.json");
+  const dataDir = process.env.DATA_DIR || "/data";
+  const configPath = path.join(dataDir, "config.json");
   const raw = fs.readFileSync(configPath, "utf8");
   return JSON.parse(raw);
 }
@@ -310,11 +314,11 @@ async function queryCollectionData(args = {}) {
   );
 }
 
-module.exports = {
+export {
   addDocuments,
   addDocumentsToolDefinition,
   listCollectionData,
   listCollectionDataToolDefinition,
   queryCollectionData,
   queryCollectionDataToolDefinition,
-};
+};