import { config } from '@config/index.js'; import axios from 'axios'; import * as cheerio from 'cheerio'; interface ScrapedChannelData { name: string; description: string; iconUrl: string; followerCount: number; inviteLink: string; } export class WhatsAppChannelScraper { private readonly userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'; private readonly delay = config.SCRAPER_INTERVAL_MS; async scrape(channelUrl: string): Promise { // Add delay to respect rate limit await this.sleep(this.delay); const response = await axios.get(channelUrl, { headers: { 'User-Agent': this.userAgent, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8', 'Accept-Language': 'en-US,en;q=0.5', 'Accept-Encoding': 'gzip, deflate, br', 'Connection': 'keep-alive', 'Upgrade-Insecure-Requests': '1', }, timeout: 15000, maxRedirects: 5, }); const $ = cheerio.load(response.data); // Extract channel data from WhatsApp web page const name = this.extractName($); const description = this.extractDescription($); const iconUrl = this.extractIcon($); const followerCount = this.extractFollowers($); return { name, description, iconUrl, followerCount, inviteLink: channelUrl, }; } private extractName($: cheerio.CheerioAPI): string { // Try multiple selectors for channel name const selectors = [ 'h1[data-testid="channel-name"]', 'h1._amgw', 'header h1', 'div[role="heading"] h1', 'h1', ]; for (const selector of selectors) { const element = $(selector).first(); if (element.length && element.text().trim()) { return element.text().trim(); } } // Fallback: try to get from meta tags const ogTitle = $('meta[property="og:title"]').attr('content'); if (ogTitle) return ogTitle; const twitterTitle = $('meta[name="twitter:title"]').attr('content'); if (twitterTitle) return twitterTitle; return 'Canal WhatsApp'; } private extractDescription($: cheerio.CheerioAPI): string { const selectors = [ 'div[data-testid="channel-description"]', 'div._amgx', 'header + div', 'meta[property="og:description"]', 'meta[name="twitter:description"]', ]; for (const selector of selectors) { const element = $(selector).first(); if (element.length) { if (element.is('meta')) { const content = element.attr('content'); if (content) return content.trim(); } else { const text = element.text().trim(); if (text) return text; } } } return ''; } private extractIcon($: cheerio.CheerioAPI): string { const selectors = [ 'img[data-testid="channel-avatar"]', 'header img[alt]', 'img._amgy', 'meta[property="og:image"]', 'meta[name="twitter:image"]', ]; for (const selector of selectors) { const element = $(selector).first(); if (element.length) { if (element.is('meta')) { const content = element.attr('content'); if (content) return content; } else { const src = element.attr('src') || element.attr('data-src'); if (src) return src; } } } return ''; } private extractFollowers($: cheerio.CheerioAPI): number { const selectors = [ 'span[data-testid="channel-followers"]', 'span._amgz', 'header span:contains("abonnés")', 'header span:contains("followers")', 'header span:contains("follower")', ]; for (const selector of selectors) { const element = $(selector).first(); if (element.length) { const text = element.text().trim(); const parsed = this.parseNumber(text); if (parsed > 0) return parsed; } } // Try to find in page text const bodyText = $('body').text(); const patterns = [ /(\d[\d\s,.]*)\s*abonnés?/i, /(\d[\d\s,.]*)\s*followers?/i, /"follower_count":\s*(\d+)/, ]; for (const pattern of patterns) { const match = bodyText.match(pattern); if (match) { return this.parseNumber(match[1]); } } return 0; } private parseNumber(text: string): number { const cleaned = text.replace(/[^\d.,KkMm]/g, '').replace(',', '.').trim(); if (!cleaned) return 0; const multiplier = cleaned.toLowerCase().includes('k') ? 1000 : cleaned.toLowerCase().includes('m') ? 1000000 : 1; const num = parseFloat(cleaned.replace(/[KkMm]/g, '')); return Math.round(num * multiplier); } private sleep(ms: number): Promise { return new Promise(resolve => setTimeout(resolve, ms)); } } export const channelScraper = new WhatsAppChannelScraper();