const fs = require('fs') const path = require('path') const { createClient } = require('./httpClient') const cheerio = require('cheerio') const patternLib = require('./patternLibrary') const { analyzePage } = require('./pageAnalyzer') const servicesDir = path.join(__dirname, '..', 'services') const probeClient = createClient({ headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36' }, timeout: 10000, }) const domainLocks = new Map() const MIN_INTERVAL_MS = 1500 async function throttledProbe(url) { let domain try { domain = new URL(url).hostname } catch { domain = url } const lastCall = domainLocks.get(domain) || 0 const elapsed = Date.now() - lastCall if (elapsed < MIN_INTERVAL_MS) { await new Promise(r => setTimeout(r, MIN_INTERVAL_MS - elapsed)) } domainLocks.set(domain, Date.now()) return probeUrl(url) } function resetThrottle() { domainLocks.clear() } const SEARCH_TERMS = ['Naruto', 'The Matrix'] function getBaseUrl(code) { // Buscar BASE_URL (patrón estándar) let match = code.match(/BASE_URL\s*=\s*['"]([^'"]+)['"]/) if (match) return match[1] // Buscar variables con URL base (detailBase, searchBase, apiUrl, siteUrl, etc.) // Soporta comillas simples, dobles y backticks match = code.match(/(?:detail|search|api|site|base)(?:[Uu]rl|[Bb]ase)\s*=\s*['"`]https?:\/\/[^'"`]+['"`]/) if (match) { const urlMatch = match[0].match(/https?:\/\/[^'"`]+/) if (urlMatch) return urlMatch[0] } return null } function extractUrlPatterns(code) { const patterns = [] // Buscar template literals con ${variable} que contengan paths de URLs const re = /`\$\{[^}]+\}([^`]*)`/g let m while ((m = re.exec(code)) !== null) { const path = m[1] if (path.length > 1) { // ignorar paths vacíos const isSearch = path.includes('?') || path.includes('q=') patterns.push({ path, isSearch, hasVariables: path.includes('${') }) } } return patterns } const MAX_HTML_SIZE = 50000 // 50KB max per page to prevent OOM async function probeUrl(url) { try { const resp = await probeClient.get(url, { validateStatus: () => true }) const rawHtml = resp.data const html = typeof rawHtml === 'string' ? rawHtml.slice(0, MAX_HTML_SIZE) : rawHtml return { url, status: resp.status, ok: resp.status === 200, html, length: typeof rawHtml === 'string' ? rawHtml.length : 0 } } catch (err) { return { url, status: null, ok: false, html: null, length: 0, error: err.message } } } function stripPath(url) { const parsed = new URL(url) const segments = parsed.pathname.split('/').filter(Boolean) if (segments.length <= 1) return null segments.pop() return `${parsed.origin}/${segments.join('/')}` } async function searchSite(baseUrl, searchPath, query) { const url = baseUrl + searchPath .replace(/\$\{encodeURIComponent\([^)]+\)\}/g, encodeURIComponent(query)) .replace(/\$\{[^}]+\}/g, encodeURIComponent(query)) try { const resp = await probeClient.get(url) const links = [] // Handle JSON responses (Next.js data API, etc.) if (typeof resp.data === 'object' && resp.data !== null) { const movies = resp.data.pageProps?.movies || resp.data.results || [] for (const m of movies.slice(0, 5)) { const slug = m.url?.slug || m.slug || '' const title = m.titles?.name || m.title || m.name || slug.split('/').pop() if (slug) { // Construir URL de detalle en el dominio principal (no en el API domain) // wv3.cuevana3.eu → www.cuevana3.eu, api.example.com → www.example.com const mainDomain = baseUrl.replace(/https?:\/\/(?:[^.]+\.)?([^/]+)(?:\/.*)?/, (_, domain) => `https://www.${domain}`).replace(/\/_next\/data\/[^/]+/, '') const slugParts = slug.split('/') const slugName = slugParts[slugParts.length - 1] // Detectar tipo: movies/ → /pelicula/, serie/ → /serie/ const isMovie = slug.startsWith('movies/') || slug.includes('/movies/') const pathPrefix = isMovie ? '/pelicula' : '/serie' const href = `${mainDomain}${pathPrefix}/${slugName}` links.push({ title, href, slug, tmdbId: m.TMDbId }) } } return { url, links: links.slice(0, 5), html: null, json: resp.data } } // Handle HTML responses const $ = cheerio.load(resp.data) $('a').each((_, el) => { const href = $(el).attr('href') || '' if (href.includes('/anime/') || href.includes('/movie/') || href.includes('/pelicula/') || href.includes('/serie/')) { const title = $(el).text().trim() || $(el).find('h1,h2,h3').text().trim() || href.split('/').pop() if (title && !links.find(l => l.href === href)) links.push({ title, href: href.startsWith('http') ? href : baseUrl + href }) } }) $('li a').each((_, el) => { const href = $(el).attr('href') || '' if (href.includes('/anime/') || href.includes('/movie/') || href.includes('/pelicula/') || href.includes('/serie/')) { const title = $(el).text().trim() || $(el).find('h1,h2,h3,h4').text().trim() if (title && !links.find(l => l.href === href)) links.push({ title: title || href.split('/').pop(), href: href.startsWith('http') ? href : baseUrl + href }) } }) return { url, links: links.slice(0, 5), html: resp.data } } catch (err) { return { url, links: [], html: null, error: err.message } } } async function tryDirectPatterns(baseUrl, mediaType, title, slug, tmdbId) { const results = [] // Only try patterns that can be constructed from title data (not search_result-dependent) const patterns = patternLib.getDirectFromTitle(mediaType) for (const p of patterns.slice(0, 5)) { const vars = { slug, id: tmdbId, season: 1, episode: 1, title } const urlPath = patternLib.fillUrlTemplate(p.urlTmpl, vars) const url = baseUrl.endsWith('/') ? baseUrl.slice(0, -1) + urlPath : baseUrl + urlPath const pr = await throttledProbe(url) results.push({ pattern: p.id, url, status: pr.status, ok: pr.ok, html: pr.ok ? pr.html : null, length: pr.length, }) if (pr.ok && results.filter(r => r.ok).length >= 1) break } return results } function sanitizeToSlug(title) { return title .toLowerCase() .replace(/[áäàâ]/g, 'a').replace(/[éëèê]/g, 'e').replace(/[íïìî]/g, 'i') .replace(/[óöòô]/g, 'o').replace(/[úüùû]/g, 'u').replace(/ñ/g, 'n') .replace(/[^a-z0-9]+/g, '-') .replace(/^-|-$/g, '') } const PROVIDER_MEDIA_TYPES = { animejl: 'anime', animeonline: 'anime', sololatino: 'anime', 'stremio-anime': 'anime', animefenix: 'anime', jkanime: 'anime', aniwave: 'anime', cinecalidadrs: 'movie', cinecalidadec: 'movie', cuevana3: 'movie', peliplus: 'movie', homecine: 'serie', } async function probe(providerName) { const servicePath = path.join(servicesDir, `${providerName}Service.js`) if (!fs.existsSync(servicePath)) return { error: `Service ${providerName} no encontrado` } const code = fs.readFileSync(servicePath, 'utf-8') const baseUrl = getBaseUrl(code) if (!baseUrl) return { error: 'No se encontró BASE_URL en el código' } const mediaType = PROVIDER_MEDIA_TYPES[providerName] || 'movie' const patterns = extractUrlPatterns(code) const results = [] let realContentUrls = [] let patternHits = [] let searchDone = false // --- FASE 1: PROBAR PATRONES DIRECTOS (sin search) --- for (const term of SEARCH_TERMS.slice(0, 1)) { const slug = sanitizeToSlug(term) const directResults = await tryDirectPatterns(baseUrl, mediaType, term, slug, 603) for (const dr of directResults) { results.push({ label: `direct(${dr.pattern})`, url: dr.url, type: 'direct', status: dr.status, ok: dr.ok, length: dr.length, html: dr.html, }) if (dr.ok) patternHits.push(dr) } } // Si patrón directo funciona, extraer contenido if (patternHits.length > 0) { for (const hit of patternHits.slice(0, 2)) { // Intentar extraer episodios si es serie/anime if (mediaType === 'anime' || mediaType === 'serie') { const episodeUrls = [ hit.url + '/episodio-1', hit.url + '/1', hit.url + '/episode-1', hit.url.replace(/\/$/, '') + '/temporada-1-capitulo-1', hit.url.replace(/\/$/, '') + '-temporada-1-capitulo-1', ] for (const epUrl of episodeUrls.slice(0, 3)) { const pr = await throttledProbe(epUrl) results.push({ label: `episode_direct(${epUrl.split('/').pop()})`, url: epUrl, type: 'episode', status: pr.status, ok: pr.ok, length: pr.length, html: pr.ok ? pr.html : null, }) } } // Extraer estructura del HTML de detalle if (hit.html) { const structureHints = patternLib.matchByExtractionPattern(hit.html) results.push({ label: `structure_hints(${structureHints.length} found)`, url: hit.url, type: 'structure', status: 200, ok: true, length: hit.html.length, html: null, hints: structureHints.map(h => ({ patternId: h.patternId, confidence: h.confidence })), }) } } } // --- FASE 2: SEARCH FALLBACK (solo si directo no funcionó) --- const directFailed = patternHits.length === 0 const noEpisodeData = !results.some(r => r.type === 'episode' && r.ok) if (directFailed || noEpisodeData) { searchDone = true const searchPattern = patterns.find(p => p.isSearch) if (searchPattern) { for (const term of SEARCH_TERMS) { const sr = await searchSite(baseUrl, searchPattern.path, term) results.push({ label: `search(${term})`, url: sr.url, type: 'search', status: sr.error ? null : 200, ok: sr.links.length > 0 || !!sr.json, // OK si hay links o JSON válido length: sr.html ? sr.html.length : 0, itemsFound: sr.links.length, error: sr.error || null, html: sr.html || null, json: sr.json || null, }) realContentUrls.push(...sr.links.map(l => ({ title: l.title, href: l.href }))) } // Probar detail + episode desde search results if (realContentUrls.length > 0) { const unique = [...new Set(realContentUrls.map(r => r.href))].slice(0, 2) for (const u of unique) { const pr = await throttledProbe(u) results.push({ label: `detail_search(${u.split('/').pop()})`, url: u, type: 'detail', status: pr.status, ok: pr.ok, length: pr.length, html: pr.ok ? pr.html : null, }) if (pr.ok && (mediaType === 'anime' || mediaType === 'serie')) { for (const suffix of ['/episodio-1', '/1']) { const ep = await throttledProbe(u + suffix) results.push({ label: `episode_search(${suffix})`, url: u + suffix, type: 'episode', status: ep.status, ok: ep.ok, length: ep.length, html: ep.ok ? ep.html : null, }) } for (const suffix of ['/episodio-1', '/1']) { const epResult = await throttledProbe(u + suffix) if (!epResult.ok) { let parentUrl = stripPath(u + suffix) let depth = 0 while (parentUrl && depth < 5) { const pr = await throttledProbe(parentUrl) results.push({ label: `hierarchy(${parentUrl.split('/').pop()})`, url: parentUrl, type: 'hierarchy', status: pr.status, ok: pr.ok, length: pr.length, html: pr.ok ? pr.html : null, depth, }) if (pr.ok) break parentUrl = stripPath(parentUrl) depth++ } } } } } } } } // Collect working HTML const workingPages = results.filter(r => r.ok && r.html) const workingHtml = workingPages.map(p => ({ url: p.url, html: p.html })) // Try structure matching on working pages (for LLM context) let structureMatches = [] for (const wp of workingPages.slice(0, 3)) { const matches = patternLib.matchByExtractionPattern(wp.html) structureMatches.push(...matches.map(m => ({ url: wp.url, patternId: m.patternId, confidence: m.confidence, matchType: m.matchType, }))) } // Learn from successful direct patterns if (patternHits.length > 0) { for (const hit of patternHits.slice(0, 1)) { const domain = new URL(baseUrl).hostname const existing = patternLib.matchBySite(baseUrl) if (existing.length === 0) { patternLib.learnFromFix( providerName, baseUrl, mediaType, hit.url.replace(baseUrl, ''), 'auto_probed', hit.html || '' ) } } } const workingSlugs = SEARCH_TERMS.map(t => ({ title: t, slug: sanitizeToSlug(t), id: t === 'Naruto' ? 462 : t === 'The Matrix' ? 603 : null })).filter(s => results.some(r => r.ok && (r.url.includes(s.slug) || r.url.includes(encodeURIComponent(s.title))))) for (const wp of workingHtml) { try { const analysis = await analyzePage(wp.html, wp.url) if (!analysis.contentHints.hasRealContent && wp.html.length < 100000) { const idx = results.findIndex(r => r.url === wp.url && r.ok) if (idx !== -1) { results[idx] = { ...results[idx], ok: false, emptyContent: true } patternHits = patternHits.filter(h => h.url !== wp.url) } } } catch {} } let pageAnalysis = null const homepageHtml = workingHtml.find(h => new URL(h.url).pathname === '/' || new URL(h.url).pathname === '') let bestHtml = workingHtml.find(h => h.html.length >= 5000 && results.some(r => r.url === h.url && r.ok)) if (!bestHtml) bestHtml = workingHtml[0] if (bestHtml) { try { pageAnalysis = await analyzePage(bestHtml.html, bestHtml.url) if (homepageHtml && homepageHtml.url !== bestHtml.url) { const homeAnalysis = await analyzePage(homepageHtml.html, homepageHtml.url) pageAnalysis.homepageType = homeAnalysis.pageType pageAnalysis.homepagePatterns = homeAnalysis.linkPatterns } } catch {} } return { provider: providerName, baseUrl, mediaType, urlResults: results.map(r => ({ label: r.label, url: r.url, type: r.type, status: r.status, ok: r.ok, length: r.length, itemsFound: r.itemsFound, error: r.error, depth: r.depth, hints: r.hints, })), hierarchyResults: results.filter(r => r.type === 'hierarchy') .map(r => ({ url: r.url, status: r.status, ok: r.ok, depth: r.depth })), workingHtml, structureMatches, patternHits: patternHits.map(h => ({ url: h.url, pattern: h.pattern, status: h.status })), searchDone, directHit: patternHits.length > 0, workingSlugs: workingSlugs.length > 0 ? workingSlugs : null, pageAnalysis, } } module.exports = { probe, throttledProbe, resetThrottle }