prefix_code:
const { generateText } = require("ai");
const { google } = require('./llm')
const fs = require("fs");
const path = require("path");

const prompts = {
  system:
    "You are a JS and opensource-software specialist, going to explain some uglified JS code.",
  user: (
    code
  ) => `Here are some uglified codes. Read and answer the following questions

1. Is it part of several open source projects code you know? If it is likely, tell me the name of the open source project.
2. The purpose of the code snippet.

Response JSON Format:
{
  "ossProjects": ["$name_1", "$name_2"],
  "purpose": "$description"
}

\`\`\`
${code}
\`\`\``,
};

const chunksDir = path.resolve(__dirname, "../chunks");

(async () => {

suffix_code:
})();

function parseMessageToJson(input) {
  // Regular expression to match JSON code blocks
  const jsonCodeBlockRegex = /```json\n([\s\S]*?)\n```/g;

  // Find all matches for JSON code blocks
  const matches = Array.from(input.matchAll(jsonCodeBlockRegex));

  if (matches.length > 1) {
    throw new Error("Multiple JSON code blocks found in the input string.");
  }

  let jsonString;

  if (matches.length === 1) {
    // Extract JSON content from the code block, trimming whitespace
    jsonString = matches[0][1].trim();
  } else {
    // No JSON code block found, use the entire input
    jsonString = input.trim();
  }

  try {
    // Parse the JSON string into an object
    return JSON.parse(jsonString);
  } catch (error) {
    throw new Error("Failed to parse JSON: " + error + "\n\n" + jsonString);
  }
}

module.exports = {
  google,
  parseMessageToJson,
  prompts
}

middle_code:
for (const chunk of fs.readdirSync(chunksDir)) {
    const mjsPath = path.resolve(chunksDir, chunk);
    if (!mjsPath.endsWith(".mjs") || chunk === 'cli.chunks.mjs') {
      continue;
    }
    const jsonPath = mjsPath.replace(".mjs", ".json");
    if (fs.existsSync(jsonPath)) {
      continue
    }
    const content = fs.readFileSync(mjsPath, "utf-8");
    const { text } = await generateText({
      model: google("gemini-2.0-flash"),
      messages: [
        {
          role: "system",
          content: prompts.system,
        },
        {
          role: "user",
          content: prompts.user(content),
        },
      ],
      maxRetries: 5
    });
    const struct = parseMessageToJson(text)
    fs.writeFileSync(jsonPath, JSON.stringify(struct, null, 2));
  }

language_type: javascript

context_code:
const { generateText } = require("ai");
const { prompts } = require("./learn-chunks");
const { google } = require("./llm");
const fs = require("fs");
const path = require("path");
const marked = require("marked");
const { markedTerminal } = require("marked-terminal");
const readline = require("readline");

marked.use(markedTerminal());

const rl = readline.createInterface({
  input: process.stdin,
  output: process.stdout,
});

const mergedChunksDir = path.resolve(__dirname, "../merged-chunks");

(async () => {
  // Prompt the user for a question
  const QUESTION = await new Promise((resolve) => {
    rl.question("Enter your question about the code: ", (answer) => {
      resolve(answer);
    });
  });

  console.log(`Analyzing question: "${QUESTION}"\n`);

  let insight = "";
  let final = "";

  for (const chunk of fs.readdirSync(mergedChunksDir)) {
    if (!chunk.startsWith("improved-")) {
      continue;
    }

    console.log("Processing", chunk);

    prompts.user = (
      code
    ) => `I am analyzing a Node.JS CLI AI program called claude-code, which implements an AI coder by interacting with LLM, using tool calling, ModelContextProtocol and other functions.

The question I want to focus on now is:

<focus_question>
${QUESTION}
</focus_question>

The following is its source code fragment, which has been obfuscated by uglify. Please tell me the code logic that is directly or indirectly related to the problem I want to study.
Only answer me what you read from the code I provided.

The code logic I am more concerned about includes:

1. Its prompt design
2. Its LLM calling method
3. Other related logic

<insight_from_other_code_fragment>
${insight}
</insight_from_other_code_fragment>

<current_code_fragment>
${code}
</current_code_fragment>
\`\`\``;

    const { text } = await generateText({
      model: google("gemini-2.0-flash"),
      messages: [
        {
          role: "system",
          content: prompts.system,
        },
        {
          role: "user",
          content: prompts.user(
            fs.readFileSync(path.resolve(mergedChunksDir, chunk), "utf-8")
          ),
        },
      ],
      maxRetries: 5,
    });

    insight += `\n  <insight_from_${chunk}>
${text}
  </insight_from_${chunk}>\n`;
    final = text;
  }

  console.log(`=== Answer ===\n`);
  console.log(marked.parse(final));

  rl.close();
})();

context_code:
const fs = require("fs");
const path = require("path");

const chunksDir = path.resolve(__dirname, "../chunks");
const mergedChunksDir = path.resolve(__dirname, "../merged-chunks");

const mergedIndex = {};

function manualFixNaming(project) {
  // fix LLM naming issue
  let lowerCaseProject = project.toLowerCase();

  return (
    {
      "sentry-javascript": "sentry",
      "react devtools": "react",
      "aws-sdk-js-v3": "aws-sdk",
      "@aws-sdk/signature-v4": "aws-sdk",
      "@smithy/smithy-client": "aws-sdk",
      "aws sdk for javascript": "aws-sdk",
      "statsig-js": "statsig",
      // 58
      "claude-code": "claude-code-1",
      // 59
      "langchain.js": 'claude-code-2',
      // 60
      zod: 'claude-code-3',
      // 61
      "claude code": "claude-code-4",
      // 62
      "continue code": "claude-code-5",
      tengu: "claude-code-5",
      // 64
      "@anthropic-ai/claude-code": "claude-code-6",
    }[lowerCaseProject] || lowerCaseProject
  );
}

for (const chunk of fs.readdirSync(chunksDir)) {
  const chunkPath = path.resolve(chunksDir, chunk);
  if (!chunkPath.endsWith(".json")) {
    continue;
  }

  const { ossProjects = [] } = JSON.parse(
    fs.readFileSync(chunkPath, "utf-8")
  );

  for (const prpject of ossProjects) {
    const projectName = manualFixNaming(prpject);
    mergedIndex[projectName] = mergedIndex[projectName] || [];
    mergedIndex[projectName].push(path.relative(chunksDir, chunkPath));
  }
}

for (const project in mergedIndex) {
  let content = "";
  for (const chunkPath of new Set(mergedIndex[project])) {
    content += fs.readFileSync(
      path.resolve(chunksDir, chunkPath.replace(".json", ".mjs")),
      "utf-8"
    );
  }

  fs.writeFileSync(
    path.resolve(mergedChunksDir, `${project.replaceAll("/", "__")}.mjs`),
    content
  );
}

fs.writeFileSync(
  path.resolve(mergedChunksDir, "chunks.index.json"),
  JSON.stringify(Object.keys(mergedIndex).reduce((prev, cur) => {
    for (const chunk of mergedIndex[cur]) {
      prev[chunk] = cur;
    }
    return prev;
  }, {}), null, 2)
);

context_code:
const acorn = require("acorn");
const walk = require("acorn-walk");
const fs = require("fs");
const path = require("path");

const CHUNK_THRESHOLD = 100_000;
const currentChunk = {
  pathCount: 1,
  content: "",
};

const source = fs.readFileSync(
  path.resolve(__dirname, "../cli.beautify.mjs"),
  "utf-8"
);

const ast = acorn.parse(source, {
  allowHashBang: true,
  sourceType: "module",
  ecmaVersion: "latest",
});

const chunksDir = path.resolve(__dirname, "../chunks");

const chunksEntry = path.resolve(chunksDir, "cli.chunks.mjs");
fs.writeFileSync(chunksEntry, "");

// help Mac users
const namedSet = new Set();
function namer(value) {
  let count = 0;
  let newValue = `${value.toLowerCase()}_${count}`;
  while (namedSet.has(newValue)) {
    count++;
    newValue = `${value.toLowerCase()}_${count}`;
  }
  namedSet.add(newValue);
  return newValue;
}

const chunksIndex = {};

walk.fullAncestor(ast, (node, _state, ancestors) => {
  if (node.type === "Program") {
    return;
  }

  const lastAncestor = ancestors[ancestors.length - 2]; // exclude current
  if (lastAncestor.type !== "Program") {
    // only split top level
    return;
  }

  let splited = false;
  switch (node.type) {
    case "FunctionDeclaration":
      splited = writeWithCheck(getContent(node), node.id.name);
      return;
    case "ExportNamedDeclaration": {
      let exportName = node.specifiers.find((s) => s.type === "ExportSpecifier")
        .exported.name;
      splited = writeWithCheck(getContent(node), exportName);
      return;
    }
    case "VariableDeclaration":
      if (node.declarations.some((decl) => decl.id.type !== "Identifier")) {
        return;
      }
      node.declarations.forEach((decl) => {
        declSplited = writeWithCheck(getContent(decl), decl.id.name);
      });
      return;
    case "ClassDeclaration":
      splited = writeWithCheck(getContent(node), node.id.name);
      return;
    default:
  }

  if (!splited) {
    appendToEntry(node);
  }
});

// flush last chunk
fs.writeFileSync(
  path.resolve(chunksDir, `chunks.${currentChunk.pathCount}.mjs`),
  currentChunk.content
);

fs.writeFileSync(
  path.resolve(chunksDir, `chunks.index.json`),
  JSON.stringify(chunksIndex, null, 2)
);

function getContent(node) {
  return `\n// @from(${getPositionInfo(node)})\n${source.slice(
    node.start,
    node.end
  )}`;
}

function writeWithCheck(content, name) {
  const chunkPath = path.resolve(chunksDir, `chunks.${currentChunk.pathCount}.mjs`);
  if (currentChunk.content.length + content.length > CHUNK_THRESHOLD) {
    // flush
    fs.writeFileSync(chunkPath, currentChunk.content);
    currentChunk.content = "";
    currentChunk.pathCount++;
  }

  // keep small chunks inline
  // if (content.length < 2000) {
  //   return false;
  // }

  if (chunksIndex[name]) {
    throw new Error("name conflict", { name });
  }
  chunksIndex[name] = path.relative(chunksDir, chunkPath);

  currentChunk.content += content;
  return true;
}

function appendToEntry(node) {
  fs.appendFileSync(
    chunksEntry,
    `\n// @from(${getPositionInfo(node)})\n` + getContent(node) + "\n"
  );
}

function getPositionInfo(node) {
  return node.loc
    ? `Ln ${node.loc.start.line}, Col ${node.loc.start.column}`
    : `Start ${node.start}, End ${node.end}`;
}

context_code:
const fs = require("fs");
const path = require("path");
const acorn = require("acorn-loose");
const walk = require("acorn-walk");

const chunksDir = path.resolve(__dirname, "../chunks");
const mergedChunksDir = path.resolve(__dirname, "../merged-chunks");
const chunksIndex = JSON.parse(
  fs.readFileSync(path.resolve(chunksDir, "chunks.index.json"), "utf-8")
);
const mergedIndex = JSON.parse(
  fs.readFileSync(path.resolve(mergedChunksDir, "chunks.index.json"), "utf-8")
);

for (const chunk of fs.readdirSync(mergedChunksDir)) {
  if (chunk.startsWith("claude-code-")) {
    const content = fs.readFileSync(
      path.resolve(mergedChunksDir, chunk),
      "utf-8"
    );

    let depsSummary =
      "/* Some variables in the code import from the following projects\n";
    const deps = {};
    for (const v of roughFindUndefinedVariables(content)) {
      if (chunksIndex[v]) {
        const project = mergedIndex[chunksIndex[v].replace(".mjs", ".json")];
        if (project.startsWith("claude-code")) {
          continue;
        }
        deps[project] = deps[project] || [];
        deps[project].push(v);
      }
    }
    for (const project in deps) {
      depsSummary += `* ${deps[project].join(
        ", "
      )} import from OSS "${project}"\n`;
    }
    depsSummary += "*/";

    fs.writeFileSync(
      path.resolve(mergedChunksDir, `improved-${chunk}`),
      `${depsSummary}\n${content}`
    );
  }
}

function roughFindUndefinedVariables(code) {
  const ast = acorn.parse(code, {
    allowHashBang: true,
    sourceType: "module",
    ecmaVersion: "latest",
  });

  const defined = new Set();
  const used = new Set();

  walk.simple(ast, {
    VariableDeclarator(node) {
      if (node.id.type === "Identifier") {
        defined.add(node.id.name);
      }
    },
    FunctionDeclaration(node) {
      if (node.id && node.id.type === "Identifier") {
        defined.add(node.id.name);
      }
    },
    ClassExpression(node) {
      if (node.id && node.id.type === "Identifier") {
        defined.add(node.id.name);
      }
    },
    Identifier(node) {
      used.add(node.name);
    },
  });

  const undefinedVars = [...used].filter((name) => !defined.has(name));
  return undefinedVars;
}

context_code:
const { createGoogleGenerativeAI } = require("@ai-sdk/google");
const { fetch, ProxyAgent } = require("undici");

const dispatcher = process.env.http_proxy ? new ProxyAgent(process.env.http_proxy) : undefined;

const google = createGoogleGenerativeAI({
  fetch: async (req, options) => {
    return fetch(req, {
      ...options,
      ...(dispatcher && { dispatcher }),
    });
  },
});

module.exports = {
  google,
};

context_code:
const beautify = require('js-beautify/js').js
const fs = require('fs')
const path = require('path')

const original = fs.readFileSync(path.resolve(__dirname, '../cli.mjs'), 'utf-8')

fs.writeFileSync(path.resolve(__dirname, '../cli.beautify.mjs'), beautify(original, {
  indent_size: 2,
  space_in_empty_paren: true,
}))

code_description: This code segment functions to: Iterate through specified files in a directory, generate AI-powered analysis of their contents using predefined prompts, extract structured data from the responses, and persist the processed results as JSON files while avoiding redundant processing of previously analyzed files.

inference_result:
```javascript
[TASK_BEGIN]
  const files = await fs.promises.readdir(chunksDir);
  const results = [];

  for (const file of files) {
    const code = await fs.promises.readFile(path.join(chunksDir, file), "utf8");
    const { text } = await generateText({
      model: google,
      systemPrompt: prompts.system,
      prompt: prompts.user(code),
    });

    const analysis = parseMessageToJson(text);
    results.push({
      file,
      code,
      analysis
    });
  }

  await fs.promises.writeFile(
    path.join(chunksDir, "results.json"),
    JSON.stringify(results, null, 2)
  );
[TASK_END]
```


prefix_code:
const fs = require("fs");
const path = require("path");

const chunksDir = path.resolve(__dirname, "../chunks");
const mergedChunksDir = path.resolve(__dirname, "../merged-chunks");

const mergedIndex = {};

function manualFixNaming(project) {
  // fix LLM naming issue
  let lowerCaseProject = project.toLowerCase();

  return (
    {
      "sentry-javascript": "sentry",
      "react devtools": "react",
      "aws-sdk-js-v3": "aws-sdk",
      "@aws-sdk/signature-v4": "aws-sdk",
      "@smithy/smithy-client": "aws-sdk",
      "aws sdk for javascript": "aws-sdk",
      "statsig-js": "statsig",
      // 58
      "claude-code": "claude-code-1",
      // 59
      "langchain.js": 'claude-code-2',
      // 60
      zod: 'claude-code-3',
      // 61
      "claude code": "claude-code-4",
      // 62
      "continue code": "claude-code-5",
      tengu: "claude-code-5",
      // 64
      "@anthropic-ai/claude-code": "claude-code-6",
    }[lowerCaseProject] || lowerCaseProject
  );
}

for (const chunk of fs.readdirSync(chunksDir)) {
  const chunkPath = path.resolve(chunksDir, chunk);
  if (!chunkPath.endsWith(".json")) {
    continue;
  }

  const { ossProjects = [] } = JSON.parse(
    fs.readFileSync(chunkPath, "utf-8")
  );

  for (const prpject of ossProjects) {
    const projectName = manualFixNaming(prpject);
    mergedIndex[projectName] = mergedIndex[projectName] || [];
    mergedIndex[projectName].push(path.relative(chunksDir, chunkPath));
  }
}

suffix_code:
fs.writeFileSync(
  path.resolve(mergedChunksDir, "chunks.index.json"),
  JSON.stringify(Object.keys(mergedIndex).reduce((prev, cur) => {
    for (const chunk of mergedIndex[cur]) {
      prev[chunk] = cur;
    }
    return prev;
  }, {}), null, 2)
);

middle_code:
for (const project in mergedIndex) {
  let content = "";
  for (const chunkPath of new Set(mergedIndex[project])) {
    content += fs.readFileSync(
      path.resolve(chunksDir, chunkPath.replace(".json", ".mjs")),
      "utf-8"
    );
  }
  fs.writeFileSync(
    path.resolve(mergedChunksDir, `${project.replaceAll("/", "__")}.mjs`),
    content
  );
}

context_code:
const { generateText } = require("ai");
const { google } = require('./llm')
const fs = require("fs");
const path = require("path");

const prompts = {
  system:
    "You are a JS and opensource-software specialist, going to explain some uglified JS code.",
  user: (
    code
  ) => `Here are some uglified codes. Read and answer the following questions

1. Is it part of several open source projects code you know? If it is likely, tell me the name of the open source project.
2. The purpose of the code snippet.

Response JSON Format:
{
  "ossProjects": ["$name_1", "$name_2"],
  "purpose": "$description"
}

\`\`\`
${code}
\`\`\``,
};

const chunksDir = path.resolve(__dirname, "../chunks");

(async () => {
  for (const chunk of fs.readdirSync(chunksDir)) {
    const mjsPath = path.resolve(chunksDir, chunk);
    if (!mjsPath.endsWith(".mjs") || chunk === 'cli.chunks.mjs') {
      continue;
    }

    const jsonPath = mjsPath.replace(".mjs", ".json");
    if (fs.existsSync(jsonPath)) {
      continue
    }

    const content = fs.readFileSync(mjsPath, "utf-8");

    const { text } = await generateText({
      model: google("gemini-2.0-flash"),
      messages: [
        {
          role: "system",
          content: prompts.system,
        },
        {
          role: "user",
          content: prompts.user(content),
        },
      ],
      maxRetries: 5
    });

    const struct = parseMessageToJson(text)
    fs.writeFileSync(jsonPath, JSON.stringify(struct, null, 2));
  }
})();

function parseMessageToJson(input) {
  // Regular expression to match JSON code blocks
  const jsonCodeBlockRegex = /```json\n([\s\S]*?)\n```/g;

  // Find all matches for JSON code blocks
  const matches = Array.from(input.matchAll(jsonCodeBlockRegex));

  if (matches.length > 1) {
    throw new Error("Multiple JSON code blocks found in the input string.");
  }

  let jsonString;

  if (matches.length === 1) {
    // Extract JSON content from the code block, trimming whitespace
    jsonString = matches[0][1].trim();
  } else {
    // No JSON code block found, use the entire input
    jsonString = input.trim();
  }

  try {
    // Parse the JSON string into an object
    return JSON.parse(jsonString);
  } catch (error) {
    throw new Error("Failed to parse JSON: " + error + "\n\n" + jsonString);
  }
}

module.exports = {
  google,
  parseMessageToJson,
  prompts
}

code_description: This code segment functions to: Aggregate file contents from multiple sources into consolidated output files by iterating through grouped entries, deduplicating file references, and combining textual data into project-specific resources.

inference_result:
```javascript
[TASK_BEGIN]
for (const project of ossProjects) {
  const projectName = manualFixNaming(project);
  mergedIndex[projectName] = mergedIndex[projectName] || [];
  mergedIndex[projectName].push(path.relative(chunksDir, chunkPath));
}
[TASK_END]
```