import bytes from 'bytes'; import { Stream, ParsedStream, Addon, ParsedFile } from '../db/index.js'; import { constants, createLogger, Env, FULL_LANGUAGE_MAPPING, getLanguageDisplayName, } from '../utils/index.js'; import { config as appConfig } from '../config/index.js'; import FileParser from './file.js'; import { parseAgeString, parseDuration, extractInfoHashFromMagnet, } from './utils.js'; import { mergeParsedFiles, arrayMerge } from './merge.js'; const logger = createLogger('parser'); class StreamParser { private count = 0; get errorRegexes(): { pattern: RegExp; message: string }[] | undefined { return [ { pattern: /invalid\s+\w+\s+(account|apikey|token)/i, message: 'Invalid account or apikey or token', }, { pattern: /public\s+rate[-\s]?limit\s+exceeded/i, message: 'Public rate limit exceeded', }, ]; } protected get filenameRegex(): RegExp | undefined { return undefined; } protected get folderNameRegex(): RegExp | undefined { return undefined; } protected get sizeRegex(): RegExp | undefined { return /(\d+(\.\d+)?)\s?(KB|MB|GB|TB)/i; } protected get sizeK(): 1024 | 1000 { return 1024; } protected get seedersRegex(): RegExp | undefined { return /[๐Ÿ‘ฅ๐Ÿ‘ค]\s*(\d+)/u; } protected get indexerEmojis(): string[] { return ['๐ŸŒ', 'โš™๏ธ', '๐Ÿ”—', '๐Ÿ”Ž', '๐Ÿ”', 'โ˜๏ธ']; } protected get indexerRegex(): RegExp | undefined { return this.getRegexForTextAfterEmojis(this.indexerEmojis); } protected get ageRegex(): RegExp | undefined { return undefined; } protected getRegexForTextAfterEmojis(emojis: string[]): RegExp { return new RegExp( `(?:${emojis.join('|')})\\s*([^\\p{Emoji_Presentation}\\n]*?)(?=\\p{Emoji_Presentation}|$|\\n)`, 'u' ); } constructor(protected readonly addon: Addon) {} parse(stream: Stream): ParsedStream | { skip: true } { if (this.shouldSkip(stream)) { return { skip: true }; } stream.description = stream.description || stream.title; if (stream.url && stream.url.startsWith('magnet:')) { stream.infoHash = extractInfoHashFromMagnet(stream.url); stream.url = undefined; } let parsedStream: ParsedStream = { id: this.getRandomId(), addon: this.addon, type: 'http', proxied: this.isProxied(stream), url: this.applyUrlModifications(stream.url ?? undefined), nzbUrl: stream.nzbUrl || undefined, tarUrls: stream.tarUrls ?? undefined, tgzUrls: stream.tgzUrls ?? undefined, '7zipUrls': stream['7zipUrls'] ?? undefined, rarUrls: stream.rarUrls ?? undefined, servers: stream.servers ?? undefined, externalUrl: stream.externalUrl ?? undefined, ytId: stream.ytId ?? undefined, requestHeaders: stream.behaviorHints?.proxyHeaders?.request, responseHeaders: stream.behaviorHints?.proxyHeaders?.response, notWebReady: stream.behaviorHints?.notWebReady ?? undefined, videoHash: stream.behaviorHints?.videoHash ?? undefined, originalName: stream.name ?? undefined, originalDescription: (stream.description || stream.title) ?? undefined, }; this.raiseErrorIfNecessary(stream, parsedStream); parsedStream.error = this.getError(stream, parsedStream); if (parsedStream.error) { parsedStream.type = constants.ERROR_STREAM_TYPE; return parsedStream; } const infoStream = this.isInfoStream(stream); if (infoStream) { parsedStream.message = infoStream; parsedStream.type = constants.INFO_STREAM_TYPE; return parsedStream; } const normaliseText = (text: string) => { return text .replace( /(mkv|mp4|avi|mov|wmv|flv|webm|m4v|mpg|mpeg|3gp|3g2|m2ts|ts|vob|ogv|ogm|divx|xvid|rm|rmvb|asf|mxf|mka|mks|mk3d|webm|f4v|f4p|f4a|f4b)$/i, '' ) .replace(/[^\p{L}\p{N}+]/gu, '') .toLowerCase() .trim(); }; parsedStream.filename = this.getFilename(stream, parsedStream); parsedStream.folderName = this.getFolder(stream, parsedStream); if ( parsedStream.folderName && parsedStream.filename && normaliseText(parsedStream.folderName) === normaliseText(parsedStream.filename) ) { parsedStream.folderName = undefined; } parsedStream.size = this.getSize(stream, parsedStream); parsedStream.folderSize = this.getFolderSize(stream, parsedStream); if ( parsedStream.size && parsedStream.folderSize && Math.abs(parsedStream.size - parsedStream.folderSize) / parsedStream.size < 0.05 ) { parsedStream.folderSize = undefined; } parsedStream.indexer = this.getIndexer(stream, parsedStream); parsedStream.service = this.getService(stream, parsedStream); parsedStream.duration = this.getDuration(stream, parsedStream); parsedStream.type = this.getStreamType( stream, parsedStream.service, parsedStream ); parsedStream.library = this.getInLibrary(stream, parsedStream); parsedStream.age = this.getAge(stream, parsedStream); parsedStream.bitrate = this.getBitrate(stream, parsedStream); parsedStream.message = this.getMessage(stream, parsedStream); parsedStream.parsedFile = this.getParsedFile(stream, parsedStream); parsedStream.torrent = { infoHash: stream.infoHash ?? this.getInfoHash(stream, parsedStream), seeders: this.getSeeders(stream, parsedStream), sources: stream.sources ?? undefined, fileIdx: stream.fileIdx ?? this.getFileIdx(stream, parsedStream) ?? undefined, private: this.isPrivate(stream, parsedStream), freeleech: this.isFreeleech(stream, parsedStream), }; parsedStream.extra = this.getExtras(stream, parsedStream); return parsedStream; } protected getExtras( _stream: Stream, _currentParsedStream: ParsedStream ): ParsedStream['extra'] { return undefined; } protected getRandomId(): string { return `${this.addon.instanceId}-${this.count++}`; } protected applyUrlModifications(url: string | undefined): string | undefined { if (url && appConfig.resources.streamUrlMappings) { let streamUrl; try { streamUrl = new URL(url); } catch (e) { return url; } for (const [key, value] of Object.entries( appConfig.resources.streamUrlMappings )) { if (streamUrl.origin === key) { const mappedUrl = new URL(value); streamUrl.protocol = mappedUrl.protocol; streamUrl.host = mappedUrl.host; streamUrl.port = mappedUrl.port; } } return streamUrl.toString(); } return url; } protected shouldSkip(stream: Stream): boolean { return false; } protected raiseErrorIfNecessary( stream: Stream, currentParsedStream: ParsedStream ) { if (!this.errorRegexes) { return; } for (const errorRegex of this.errorRegexes) { if (errorRegex.pattern.test(stream.description || stream.title || '')) { throw new Error(errorRegex.message); } } } protected getError( stream: Stream, currentParsedStream: ParsedStream ): ParsedStream['error'] | undefined { return undefined; } protected getFilename( stream: Stream, currentParsedStream: ParsedStream ): string | undefined { let filename = stream.behaviorHints?.filename; if (filename) { return filename; } const description = stream.description || stream.title; if (!description) { return undefined; } if (this.filenameRegex) { const match = description.match(this.filenameRegex); if (match) { return match[1]; } } // attempt to find a filename by finding the most suitable line that has more info const potentialFilenames = description .split('\n') .filter((line) => line.trim() !== '') .splice(0, 5); for (const line of potentialFilenames) { const parsed = FileParser.parse(line); if (parsed.year || parsed.seasons?.length || parsed.episodes?.length) { filename = line; break; } } if (!filename) { filename = description.split('\n')[0]; } return filename ?.trim() ?.replace(/^\p{Emoji_Presentation}+/gu, '') ?.replace(/^[^\s:]+:\s*/, ''); } protected getFolder( stream: Stream, currentParsedStream: ParsedStream ): string | undefined { if (this.folderNameRegex) { const match = stream.description?.match(this.folderNameRegex); if (match) { return match[1]; } } if (typeof stream.behaviorHints?.folderName === 'string') { return stream.behaviorHints.folderName; } return undefined; } protected getResolution( stream: Stream, currentParsedStream: ParsedStream ): string | undefined { return undefined; // } protected getReleaseGroup( stream: Stream, currentParsedStream: ParsedStream ): string | undefined { return undefined; // } protected getSize( stream: Stream, currentParsedStream: ParsedStream ): number | undefined { let description = stream.description || stream.title; if (currentParsedStream.filename && description) { description = description.replace(currentParsedStream.filename, ''); } if (currentParsedStream.folderName && description) { description = description.replace(currentParsedStream.folderName, ''); } let size = stream.behaviorHints?.videoSize || (stream as any).size || (stream as any).sizeBytes || (stream as any).sizebytes || (description && this.calculateBytesFromSizeString(description)) || (stream.name && this.calculateBytesFromSizeString(stream.name)); if (typeof size === 'string') { size = bytes.parse(size); } else if (typeof size === 'number') { size = Math.round(size); } if (Number.isFinite(size) && size > 0) { return size; } } protected getFolderSize( stream: Stream, currentParsedStream: ParsedStream ): number | undefined { if ( (stream.behaviorHints?.folderSize !== undefined && typeof stream.behaviorHints?.folderSize === 'number') || typeof stream.behaviorHints?.folderSize === 'string' ) { return ( bytes.parse(stream.behaviorHints?.folderSize.toString()) ?? undefined ); } return undefined; } protected getSeeders( stream: Stream, currentParsedStream: ParsedStream ): number | undefined { const regex = this.seedersRegex; if (!regex) { return undefined; } const match = stream.description?.match(regex); if (match) { return parseInt(match[1]); } return undefined; } protected getAge( stream: Stream, currentParsedStream: ParsedStream ): number | undefined { const regex = this.ageRegex; if (!regex) { return undefined; } const match = stream.description?.match(regex); if (match) { return parseAgeString(match[1]); } return undefined; } /** * Converts age strings like "1d", "5h", "30m", "456d" to hours * @param ageString - The age string to parse (e.g. "1d", "5h", "30m") * @returns The age in hours, or undefined if parsing fails */ protected isProxied(stream: Stream): boolean { return false; } protected isPrivate( stream: Stream, currentParsedStream: ParsedStream ): boolean | undefined { return false; } protected isFreeleech( stream: Stream, currentParsedStream: ParsedStream ): boolean | undefined { return false; } protected getIndexer( stream: Stream, currentParsedStream: ParsedStream ): string | undefined { const regex = this.indexerRegex; if (!regex) { return undefined; } const match = stream.description?.match(regex); if (match) { return match[1].trim(); } return undefined; } protected isInfoStream(stream: Stream): string | undefined { return undefined; } protected getMessage( stream: Stream, currentParsedStream: ParsedStream ): string | undefined { return undefined; } protected getService( stream: Stream, currentParsedStream: ParsedStream ): ParsedStream['service'] | undefined { return this.parseServiceData(stream.name || ''); } protected getInfoHash( stream: Stream, currentParsedStream: ParsedStream ): string | undefined { if (!stream.url) return undefined; try { return decodeURIComponent(stream.url).match( /(?:(?<=btih:)|(?<=[-/[(;:&]))[a-fA-F0-9]{40}(?=$|[-\]\)/:;&?])/ )?.[0]; } catch { return undefined; } } protected getFileIdx( stream: Stream, currentParsedStream: ParsedStream ): number | undefined { return undefined; } protected getDuration( stream: Stream, currentParsedStream: ParsedStream ): number | undefined { return parseDuration(stream.description || ''); } protected getBitrate( _: Stream, currentParsedStream: ParsedStream ): number | undefined { if ( currentParsedStream.size && currentParsedStream.duration && !currentParsedStream.bitrate ) { const sizeBits = currentParsedStream.size * 8; const durationSeconds = currentParsedStream.duration / 1000; if (durationSeconds > 0) { return Math.round(sizeBits / durationSeconds); } } return undefined; } protected getStreamType( stream: Stream, service: ParsedStream['service'], currentParsedStream: ParsedStream ): ParsedStream['type'] { if (stream.url?.endsWith('.m3u8')) { return 'live'; } if (stream.externalUrl) { return 'external'; } if (service?.id === constants.EASYNEWS_SERVICE) { return 'usenet'; } else if (service) { return 'debrid'; } if (stream.url) { return 'http'; } if (stream.infoHash) { return 'p2p'; } if (stream.ytId) { return 'youtube'; } if (stream.nzbUrl) { return 'stremio-usenet'; } if ( stream['7zipUrls']?.length || stream.rarUrls?.length || stream?.tarUrls?.length || stream.tgzUrls?.length ) { return 'archive'; } throw new Error('Invalid stream, missing a required stream property'); } protected getParsedFileMergeOverrides( stream: Stream, currentParsedStream: ParsedStream ): Partial { return {}; } /** * Parses the filename and folder name from the stream, merges the results, * and applies season-pack detection heuristics. */ protected getParsedFile( stream: Stream, parsedStream: ParsedStream ): ParsedFile | undefined { const folderParsed = parsedStream.folderName ? FileParser.parse(parsedStream.folderName) : undefined; const fileParsed = parsedStream.filename ? FileParser.parse(parsedStream.filename) : undefined; const merged = mergeParsedFiles(fileParsed, folderParsed, { // Overrides to include any info we can extract from the stream description resolution: this.getResolution(stream, parsedStream) || fileParsed?.resolution || folderParsed?.resolution, releaseGroup: this.getReleaseGroup(stream, parsedStream) || fileParsed?.releaseGroup || folderParsed?.releaseGroup, languages: arrayMerge( arrayMerge(folderParsed?.languages, fileParsed?.languages), this.getLanguages(stream, parsedStream) ), ...this.getParsedFileMergeOverrides(stream, parsedStream), }); if (!merged) return undefined; // Detect season pack based on folder size being significantly larger than file size if ( !merged.seasonPack && merged.episodes && merged.episodes.length > 0 && parsedStream.folderSize && parsedStream.size && parsedStream.folderSize > parsedStream.size * 2 ) { merged.seasonPack = true; } // Detect season pack when more than 5 episodes are present if (!merged.seasonPack && merged.episodes && merged.episodes.length > 5) { merged.seasonPack = true; } return merged; } /** * Extracts languages from the stream description using country flags. * @param stream - The stream object containing the description. * @param currentParsedStream - The current parsed stream object. * @returns An array of language strings. */ protected getLanguages( stream: Stream, currentParsedStream: ParsedStream ): string[] { const countryFlagPattern = /[\u{1F1E6}-\u{1F1FF}]{2}/gu; const descriptionMatches = stream.description?.match(countryFlagPattern); const nameMatches = stream.name?.match(countryFlagPattern); const flags = [ ...(descriptionMatches ? [...new Set(descriptionMatches)] : []), ...(nameMatches ? [...new Set(nameMatches)] : []), ]; const languages = flags .map((flag) => this.convertFlagToLanguage(flag)) .filter((language) => language !== undefined); return languages; } protected convertFlagToLanguage(flag: string): string | undefined { const possibleLanguages = FULL_LANGUAGE_MAPPING.filter( (language) => language.flag === flag ); const language = possibleLanguages.find((l) => l.flag_priority) || possibleLanguages[0]; if (!language) return undefined; const languageName = getLanguageDisplayName(language); return constants.LANGUAGES.includes(languageName as any) ? languageName : undefined; } protected convertISO6392ToLanguage(code: string): string | undefined { const lang = FULL_LANGUAGE_MAPPING.find( (language) => language.iso_639_2 === code ); return lang ? getLanguageDisplayName(lang) : undefined; } protected getInLibrary( stream: Stream, currentParsedStream: ParsedStream ): boolean { return this.addon.library ?? false; } protected calculateBytesFromSizeString( size: string, sizeRegex?: RegExp ): number | undefined { const k = this.sizeK; const sizePattern = sizeRegex || this.sizeRegex; if (!sizePattern) { return undefined; } const match = size.match(sizePattern); if (!match) return 0; const value = parseFloat(match[1]); const unit = match[3]; let result = 0; switch (unit.toUpperCase()) { case 'TB': result = value * k * k * k * k; break; case 'GB': result = value * k * k * k; break; case 'MB': result = value * k * k; break; case 'KB': result = value * k; break; default: return 0; } return Math.round(result); } protected parseServiceData( string: string ): ParsedStream['service'] | undefined { const cleanString = string.replace(/web-?dl/i, ''); const services = constants.SERVICE_DETAILS; const cachedSymbols = ['+', 'โšก', '๐Ÿš€', 'cached', '๐ŸŒฉ๏ธ', '๐Ÿ“ซ']; const uncachedSymbols = ['โณ', 'download', 'UNCACHED', 'โ˜๏ธ']; let streamService: ParsedStream['service'] | undefined; Object.values(services).forEach((service) => { // for each service, generate a regexp which creates a regex with all known names separated by | const regex = new RegExp( `(^|(? string.includes(symbol))) { cached = false; } // check if any of the cachedSymbols are in the string else if (cachedSymbols.some((symbol) => string.includes(symbol))) { cached = true; } streamService = { id: service.id, cached: cached, }; } }); return streamService; } } export default StreamParser;