Create src/scraper.js

src/scraper.js

@@ -0,0 +1,107 @@
+const axios = require('axios');
+const { db, hydrateMemory, TOOLS_COLLECTION } = require('./db');
+
+const GITHUB_API = 'https://api.github.com';
+const HEADERS = { Authorization: `token ${process.env.GITHUB_TOKEN}` };
+
+// Helper: Split array into chunks (Firestore limits batches to 500)
+const chunkArray = (arr, size) => 
+  Array.from({ length: Math.ceil(arr.length / size) }, (v, i) =>
+    arr.slice(i * size, i * size + size)
+  );
+
+// 1. The Deep Link Scanner (Monorepo aware)
+async function scanRepoContents(repo) {
+  const toolsFound = [];
+  
+  // Logic: Check if it's a monorepo by looking for 'packages' or 'servers' folder
+  // For MVP: We assume root is a tool, OR specific subfolders are tools.
+  // This is a simplified "heuristic" scanner.
+  
+  // Base Tool (The Root Repo)
+  toolsFound.push({
+    id: `gh_${repo.id}`,
+    name: repo.name,
+    description: repo.description || "No description provided.",
+    url: repo.html_url,
+    repo_url: repo.html_url,
+    install_command: `npx -y ${repo.name}`, // Placeholder prediction
+    keywords: repo.topics || [],
+    stats: {
+      stars: repo.stargazers_count,
+      forks: repo.forks_count,
+      last_updated: repo.pushed_at
+    },
+    // Initialize empty reputation/analytics
+    reputation: { score: 0, verified: false, is_malicious: false },
+    analytics: { views: 0, installs: 0, success_rate: 0 }
+  });
+
+  return toolsFound;
+}
+
+// 2. The Main Smart Scraper
+async function runSmartScraper() {
+  console.log("🕷️ Starting Smart Scrape Cycle...");
+  
+  try {
+    // A. Fetch Candidates from GitHub
+    // searching for standard tag
+    const { data } = await axios.get(`${GITHUB_API}/search/repositories?q=topic:mcp-server&sort=updated&per_page=100`, { headers: HEADERS });
+    
+    const batchOps = [];
+    const knownIds = new Set(); // To prevent duplicates
+
+    // Get current DB state to check timestamps (Optimization)
+    const existingDocs = await db.collection(TOOLS_COLLECTION).select('stats.last_updated').get();
+    const existingMap = new Map();
+    existingDocs.forEach(doc => existingMap.set(doc.id, doc.data().stats.last_updated));
+
+    // B. Process Repos
+    for (const repo of data.items) {
+      // 1. Check if update is needed (Incremental Update)
+      const cachedTime = existingMap.get(`gh_${repo.id}`);
+      const remoteTime = repo.pushed_at;
+
+      if (cachedTime && cachedTime === remoteTime) {
+        // console.log(`⏩ Skipping ${repo.name} (No changes)`);
+        continue;
+      }
+
+      // 2. Extract Tools (Deep Linking Logic)
+      const extractedTools = await scanRepoContents(repo);
+
+      extractedTools.forEach(tool => {
+        if (!knownIds.has(tool.id)) {
+            // Prepare Firestore Write
+            const docRef = db.collection(TOOLS_COLLECTION).doc(tool.id);
+            batchOps.push({ ref: docRef, data: tool });
+            knownIds.add(tool.id);
+        }
+      });
+    }
+
+    // C. Batch Write to Disk
+    if (batchOps.length > 0) {
+      console.log(`💾 Writing ${batchOps.length} updates to Firestore...`);
+      const chunks = chunkArray(batchOps, 400); // Safety limit
+      
+      for (const chunk of chunks) {
+        const batch = db.batch();
+        chunk.forEach(op => batch.set(op.ref, op.data, { merge: true }));
+        await batch.commit();
+      }
+      
+      // D. Update RAM
+      await hydrateMemory();
+      console.log("✅ Scrape & Hydrate Complete.");
+    } else {
+      console.log("💤 No updates needed.");
+    }
+
+  } catch (err) {
+    console.error("❌ Scraper Failed:", err.message);
+  }
+}
+
+module.exports = { runSmartScraper };