Update scraper.js

scraper.js

@@ -7,20 +7,25 @@ const HEADERS = {
     Accept: 'application/vnd.github.v3+json'
 };
 
-// HELPER: Extract links from Markdown READMEs
+// HELPER: Extract links from Markdown
 const extractLinksFromMarkdown = (text) => {
-    const regex = /\[([^\]]+)\]\((https:\/\/github\.com\/[^)]+)\)\s*(?:-|—)?\s*([^\n]+)?/g;
+    const regex = /\[([^\]]+)\]\((https:\/\/github\.com\/([^/)]+)\/([^/)]+))\)\s*(?:-|—)?\s*([^\n]+)?/g;
     const links = [];
     let match;
     while ((match = regex.exec(text)) !== null) {
-        links.push({ name: match[1], url: match[2], description: match[3] || "" });
+        links.push({ 
+            name: match[1], 
+            url: match[2], 
+            owner: match[3], 
+            repo: match[4], 
+            description: match[5] || "" 
+        });
     }
     return links;
 };
 
-// PHASE A: Scrape GitHub Topic (Broad Search)
+// PHASE A: Search Topics
 async function getTopicTools() {
-    console.log("🔍 Searching GitHub Topics...");
     let tools = [];
     for (let page = 1; page <= 2; page++) {
         try {
@@ -29,7 +34,7 @@ async function getTopicTools() {
                 { headers: HEADERS }
             );
             tools.push(...data.items.map(repo => ({
-                id: `gh_${repo.id}`,
+                id: `gh_${repo.owner.login}_${repo.name}`.toLowerCase(), // Normalized ID
                 name: repo.name,
                 description: repo.description,
                 url: repo.html_url,
@@ -38,15 +43,15 @@ async function getTopicTools() {
                 keywords: repo.topics || [],
                 stars: repo.stargazers_count,
                 last_updated: repo.pushed_at,
-                is_awesome_listed: false, // Default for search
+                is_awesome_listed: false,
                 discovery_source: 'github_topic'
             })));
-        } catch (e) { console.error(`Topic Page ${page} failed`); }
+        } catch (e) { console.error("Topic search error"); }
     }
     return tools;
 }
 
-// PHASE B: Scrape Awesome Lists (High Quality)
+// PHASE B: Awesome Lists
 async function getAwesomeTools() {
     const sources = [
         { owner: 'wong2', repo: 'awesome-mcp-servers' },
@@ -56,39 +61,33 @@ async function getAwesomeTools() {
 
     for (const s of sources) {
         try {
-            console.log(`🌟 Scraping Awesome List: ${s.owner}/${s.repo}`);
             const { data } = await axios.get(`${GITHUB_API}/repos/${s.owner}/${s.repo}/readme`, { headers: HEADERS });
             const content = Buffer.from(data.content, 'base64').toString();
             const links = extractLinksFromMarkdown(content);
 
             links.forEach(link => {
-                const parts = link.url.replace('https://github.com/', '').split('/');
-                if (parts.length >= 2) {
-                    tools.push({
-                        id: `gh_ext_${parts[0]}_${parts[1]}`.toLowerCase(), // Normalized ID
-                        name: link.name,
-                        description: link.description,
-                        url: link.url,
-                        repo_url: link.url,
-                        install_command: `npx -y ${parts[1]}`,
-                        is_awesome_listed: true, // THE FLAG
-                        discovery_source: 'awesome_list'
-                    });
-                }
+                tools.push({
+                    id: `gh_${link.owner}_${link.repo}`.toLowerCase(), // Same ID format as Topic Search
+                    name: link.name,
+                    description: link.description,
+                    url: link.url,
+                    repo_url: link.url,
+                    install_command: `npx -y ${link.repo}`,
+                    is_awesome_listed: true,
+                    discovery_source: 'awesome_list'
+                });
             });
-        } catch (e) { console.error(`Awesome List ${s.repo} failed`); }
+        } catch (e) { console.error(`List ${s.repo} error`); }
     }
     return tools;
 }
 
-// PHASE C: Monorepo Deep Scan
+// PHASE C: Monorepos
 async function getMonorepoTools() {
     const monorepos = [{ owner: 'modelcontextprotocol', repo: 'servers' }];
     let tools = [];
-
     for (const m of monorepos) {
-        const paths = ['src', 'servers', 'packages'];
-        for (const path of paths) {
+        for (const path of ['src', 'servers', 'packages']) {
             try {
                 const { data } = await axios.get(`${GITHUB_API}/repos/${m.owner}/${m.repo}/contents/${path}`, { headers: HEADERS });
                 if (Array.isArray(data)) {
@@ -100,7 +99,7 @@ async function getMonorepoTools() {
                             url: dir.html_url,
                             repo_url: `https://github.com/${m.owner}/${m.repo}`,
                             install_command: `npx -y @modelcontextprotocol/server-${dir.name}`,
-                            is_awesome_listed: true, // Official tools count as Awesome
+                            is_awesome_listed: true,
                             discovery_source: 'official_monorepo'
                         });
                     });
@@ -111,30 +110,40 @@ async function getMonorepoTools() {
     return tools;
 }
 
-// MAIN EXECUTION
 async function runSmartScraper() {
     console.log("🚀 Aggressive Scraper Started...");
-    
     try {
         const topicTools = await getTopicTools();
         const awesomeTools = await getAwesomeTools();
         const monoTools = await getMonorepoTools();
 
-        // Combine all results
         const allResults = [...topicTools, ...awesomeTools, ...monoTools];
 
-        // Deduplicate: If same tool is found in Search and Awesome, Awesome (with the flag) wins.
+        // --- THE FIX: DEDUPLICATE BY ID ---
         const finalMap = new Map();
+        
         allResults.forEach(item => {
-            const existing = finalMap.get(item.url); // Use URL as the key for deduping
-            if (!existing || (item.is_awesome_listed && !existing.is_awesome_listed)) {
-                finalMap.set(item.url, item);
+            const existing = finalMap.get(item.id);
+            
+            if (!existing) {
+                finalMap.set(item.id, item);
+            } else {
+                // If we find the same ID again, prefer the one that is Awesome Listed
+                if (item.is_awesome_listed && !existing.is_awesome_listed) {
+                    finalMap.set(item.id, item);
+                }
+                // Also update stars if the new object has them
+                if (item.stars > (existing.stars || 0)) {
+                    existing.stars = item.stars;
+                    finalMap.set(item.id, existing);
+                }
             }
         });
 
         const finalTools = Array.from(finalMap.values());
 
-        // Upsert to Supabase
+        console.log(`📊 Preparing to upsert ${finalTools.length} unique tools...`);
+
         const { error } = await supabase
             .from('mcp_tools')
             .upsert(finalTools, { onConflict: 'id' });
@@ -142,8 +151,6 @@ async function runSmartScraper() {
         if (error) throw error;
 
         console.log(`✅ Success! Indexed ${finalTools.length} tools.`);
-        console.log(`✨ Awesome tools found: ${finalTools.filter(t => t.is_awesome_listed).length}`);
-        
         await hydrateMemory();
 
     } catch (err) {