aiostreams / packages /core /src /streams /precomputer.ts
f4b404's picture
Upload folder using huggingface_hub
f55c62a verified
Raw
History Blame Contribute Delete
16.5 kB
import { ParsedStream, UserData } from '../db/schemas.js';
import {
createLogger,
RegexAccess,
formRegexFromKeywords,
compileRegex,
parseRegex,
} from '../utils/index.js';
import {
StreamSelector,
extractNamesFromExpression,
} from '../parser/streamExpression.js';
import { StreamContext } from './context.js';
const logger = createLogger('precomputer');
export interface PrecomputeSubTimings {
/** Time spent computing preferred regex/keyword matches (per-stream). */
preferredRegexMs: number;
/** Time spent computing ranked regex pattern scores (per-stream). */
rankedRegexMs: number;
/** Time spent evaluating ranked stream expressions (SEL). */
rankedSELMs: number;
/** Time spent evaluating preferred stream expressions (SEL). */
preferredSELMs: number;
/** Combined wall-clock time for all four stages. */
totalMs: number;
}
class StreamPrecomputer {
private userData: UserData;
private accumulatedTimings: PrecomputeSubTimings;
constructor(userData: UserData) {
this.userData = userData;
this.accumulatedTimings = {
preferredRegexMs: 0,
rankedRegexMs: 0,
rankedSELMs: 0,
preferredSELMs: 0,
totalMs: 0,
};
}
/** Returns accumulated precompute timings for this request (fetcher + pipeline runs combined). */
public getPrecomputeTimings(): PrecomputeSubTimings {
return { ...this.accumulatedTimings };
}
/** Resets accumulated timings for a new request (call once per getStreams invocation). */
public resetPrecomputeTimings(): void {
this.accumulatedTimings = {
preferredRegexMs: 0,
rankedRegexMs: 0,
rankedSELMs: 0,
preferredSELMs: 0,
totalMs: 0,
};
}
/**
* Precompute SeaDex only - runs BEFORE filtering so seadex() works in Included SEL
* Uses StreamContext's cached SeaDex data when available.
*/
public async precomputeSeaDexOnly(
streams: ParsedStream[],
context: StreamContext
): Promise<number> {
const start = Date.now();
if (!context.isAnime || this.userData.enableSeadex === false) {
return 0;
}
// Wait for SeaDex data if it's being fetched
const seadexResult = await context.getSeaDex();
if (!seadexResult) {
return Date.now() - start;
}
this.precomputeSeaDexFromResult(
streams,
seadexResult,
context.animeEntry?.mappings?.anilistId
);
return Date.now() - start;
}
/**
* Precompute preferred matches - runs AFTER filtering on fewer streams.
* When `skipPerStreamIds` is provided, per-stream operations (regex/keyword matching)
* skip streams that were already precomputed (e.g. in the fetcher).
* SEL-based operations always re-evaluate against the full stream list since
* selections can depend on the composition of the entire set.
*/
public async precomputePreferred(
streams: ParsedStream[],
context: StreamContext,
skipPerStreamIds?: Set<string>
): Promise<PrecomputeSubTimings> {
const start = Date.now();
// preferred regex / keywords --> ranked regex patterns --> ranked stream expressions --> preferred stream expressions
// this is the optimal order so that regexMatched can be used in RSE/PSE and streamExpressionScore and regexScore can be used in PSE
const preferredRegexMs = await this.precomputePreferredRegexMatches(
streams,
skipPerStreamIds
);
const rankedRegexMs = await this.precomputeRankedRegexPatterns(
streams,
skipPerStreamIds
);
const rankedSELMs = await this.precomputeRankedStreamExpressions(
streams,
context
);
const preferredSELMs = await this.precomputePreferredExpressionMatches(
streams,
context
);
const totalMs = Date.now() - start;
logger.debug(
{
took: totalMs,
skipped: skipPerStreamIds?.size ?? 0,
preferredRegexMs,
rankedRegexMs,
rankedSELMs,
preferredSELMs,
},
'precompute complete'
);
// Accumulate into per-request totals (mirrors how filterer accumulates filterTimings)
this.accumulatedTimings.preferredRegexMs += preferredRegexMs;
this.accumulatedTimings.rankedRegexMs += rankedRegexMs;
this.accumulatedTimings.rankedSELMs += rankedSELMs;
this.accumulatedTimings.preferredSELMs += preferredSELMs;
this.accumulatedTimings.totalMs += totalMs;
return {
preferredRegexMs,
rankedRegexMs,
rankedSELMs,
preferredSELMs,
totalMs,
};
}
/**
* Precompute ranked stream expression scores.
* Each stream accumulates scores from all matching expressions.
*/
private async precomputeRankedStreamExpressions(
streams: ParsedStream[],
context: StreamContext
): Promise<number> {
if (
!this.userData.rankedStreamExpressions?.length ||
streams.length === 0
) {
return 0;
}
const start = Date.now();
const selector = new StreamSelector(context.toExpressionContext());
// initialise each stream's score and match list, and build an id -> stream
// map so we can resolve back to original references after selector.select()
const streamsById = new Map<string, ParsedStream>();
for (const stream of streams) {
stream.streamExpressionScore = 0;
stream.rankedStreamExpressionsMatched = [];
streamsById.set(stream.id, stream);
}
for (const { expression, score, enabled } of this.userData
.rankedStreamExpressions) {
if (enabled === false) {
continue;
}
try {
const selectedStreams = await selector.select(streams, expression);
const exprNames = extractNamesFromExpression(expression);
for (const selected of selectedStreams) {
const stream = streamsById.get(selected.id);
if (!stream) continue;
stream.streamExpressionScore =
(stream.streamExpressionScore ?? 0) + score;
if (exprNames) {
stream.rankedStreamExpressionsMatched = [
...(stream.rankedStreamExpressionsMatched ?? []),
...exprNames,
];
}
}
} catch (error) {
logger.error(
{
expression,
err: error instanceof Error ? error.message : String(error),
},
'failed to apply ranked stream expression'
);
}
}
const nonZeroScores = streams.filter(
(s) => (s.streamExpressionScore ?? 0) !== 0
).length;
logger.debug(
{ streams: streams.length, nonZeroScores, took: Date.now() - start },
'ranked stream expressions computed'
);
return Date.now() - start;
}
private async precomputeRankedRegexPatterns(
streams: ParsedStream[],
skipStreamIds?: Set<string>
): Promise<number> {
if (!this.userData.rankedRegexPatterns?.length || streams.length === 0) {
return 0;
}
const start = Date.now();
const regexes = await Promise.all(
this.userData.rankedRegexPatterns.map(async (entry) => ({
...entry,
regex: await compileRegex(entry.pattern),
}))
);
const streamsToProcess = skipStreamIds
? streams.filter((s) => !skipStreamIds.has(s.id))
: streams;
for (const stream of streamsToProcess) {
if (!stream.filename) {
continue;
}
const matched: string[] = [];
let totalScore = 0;
for (const { regex, pattern, name, score } of regexes) {
if (
regex.test(stream.filename) ||
(stream.folderName && regex.test(stream.folderName))
) {
if (name) matched.push(name);
totalScore += score;
}
}
if (matched.length > 0) {
stream.rankedRegexesMatched = matched;
stream.regexScore = totalScore;
}
}
logger.debug(
{
matched: streams.filter((s) => s.rankedRegexesMatched?.length).length,
streams: streams.length,
took: Date.now() - start,
},
'ranked regex patterns computed'
);
return Date.now() - start;
}
/**
* Apply SeaDex tags to streams using pre-fetched SeaDex data
*/
private precomputeSeaDexFromResult(
streams: ParsedStream[],
seadexResult: {
bestHashes: Set<string>;
allHashes: Set<string>;
bestGroups: Set<string>;
allGroups: Set<string>;
},
anilistId: string | number | undefined
) {
if (
seadexResult.bestHashes.size === 0 &&
seadexResult.allHashes.size === 0 &&
seadexResult.bestGroups.size === 0 &&
seadexResult.allGroups.size === 0
) {
logger.debug({ anilistId }, 'no seadex releases found');
return;
}
logger.debug(
{
anilistId,
bestHashes: Array.from(seadexResult.bestHashes),
allHashes: Array.from(seadexResult.allHashes),
bestGroups: Array.from(seadexResult.bestGroups),
allGroups: Array.from(seadexResult.allGroups),
},
'applying seadex tags'
);
let seadexBestCount = 0;
let seadexCount = 0;
let seadexGroupFallbackCount = 0;
let anyHashMatched = false;
// First pass: try hash matching for all streams
for (const stream of streams) {
const infoHash = stream.torrent?.infoHash?.toLowerCase();
if (infoHash) {
const isBest = seadexResult.bestHashes.has(infoHash);
const isSeadex = seadexResult.allHashes.has(infoHash);
if (isSeadex) {
stream.seadex = {
isBest,
isSeadex: true,
};
if (isBest) {
seadexBestCount++;
}
seadexCount++;
anyHashMatched = true;
}
}
}
// Second pass: fallback to release group matching ONLY if no hash matched
if (!anyHashMatched) {
for (const stream of streams) {
// Skip streams already tagged
if (stream.seadex) {
continue;
}
const releaseGroup = stream.parsedFile?.releaseGroup?.toLowerCase();
if (releaseGroup) {
const isBestGroup = seadexResult.bestGroups.has(releaseGroup);
const isSeadexGroup = seadexResult.allGroups.has(releaseGroup);
if (isBestGroup || isSeadexGroup) {
stream.seadex = {
isBest: isBestGroup,
isSeadex: true,
};
if (isBestGroup) {
seadexBestCount++;
}
seadexCount++;
seadexGroupFallbackCount++;
}
}
}
}
if (seadexCount > 0) {
logger.debug(
{
tagged: seadexCount,
best: seadexBestCount,
groupFallback: seadexGroupFallbackCount,
anilistId,
},
'seadex tagging complete'
);
}
}
/**
* Precompute preferred regex, keyword, and stream expression matches.
* When `skipStreamIds` is provided, per-stream keyword and regex matching
* is skipped for those streams (they were already computed in the fetcher).
*/
private async precomputePreferredRegexMatches(
streams: ParsedStream[],
skipStreamIds?: Set<string>
): Promise<number> {
const start = Date.now();
const preferredRegexPatterns =
(await RegexAccess.isRegexAllowed(
this.userData,
this.userData.preferredRegexPatterns?.map(
(pattern) => pattern.pattern
) ?? []
)) && this.userData.preferredRegexPatterns
? await Promise.all(
this.userData.preferredRegexPatterns.map(async (pattern) => {
return {
name: pattern.name,
negate: parseRegex(pattern.pattern).flags.includes('n'),
pattern: await compileRegex(pattern.pattern),
};
})
)
: undefined;
const preferredKeywordsPatterns = this.userData.preferredKeywords
? await formRegexFromKeywords(this.userData.preferredKeywords)
: undefined;
if (
!preferredRegexPatterns &&
!preferredKeywordsPatterns &&
!this.userData.preferredStreamExpressions?.length
) {
return 0;
}
const streamsToProcess = skipStreamIds
? streams.filter((s) => !skipStreamIds.has(s.id))
: streams;
if (preferredKeywordsPatterns) {
streamsToProcess.forEach((stream) => {
stream.keywordMatched =
preferredKeywordsPatterns.test(stream.filename || '') ||
preferredKeywordsPatterns.test(stream.folderName || '') ||
preferredKeywordsPatterns.test(
stream.parsedFile?.releaseGroup || ''
) ||
preferredKeywordsPatterns.test(stream.indexer || '');
});
}
const determineMatch = (
stream: ParsedStream,
regexPattern: { pattern: RegExp; negate: boolean },
attribute?: string
) => {
return attribute ? regexPattern.pattern.test(attribute) : false;
};
if (preferredRegexPatterns) {
streamsToProcess.forEach((stream) => {
for (let i = 0; i < preferredRegexPatterns.length; i++) {
// if negate, then the pattern must not match any of the attributes
// and if the attribute is undefined, then we can consider that as a non-match so true
const regexPattern = preferredRegexPatterns[i];
const filenameMatch = determineMatch(
stream,
regexPattern,
stream.filename
);
const folderNameMatch = determineMatch(
stream,
regexPattern,
stream.folderName
);
const releaseGroupMatch = determineMatch(
stream,
regexPattern,
stream.parsedFile?.releaseGroup
);
const indexerMatch = determineMatch(
stream,
regexPattern,
stream.indexer
);
let match =
filenameMatch ||
folderNameMatch ||
releaseGroupMatch ||
indexerMatch;
match = regexPattern.negate ? !match : match;
if (match) {
stream.regexMatched = {
name: regexPattern.name,
pattern: regexPattern.pattern.source,
index: i,
};
break;
}
}
});
}
return Date.now() - start;
}
private async precomputePreferredExpressionMatches(
streams: ParsedStream[],
context: StreamContext
): Promise<number> {
const start = Date.now();
if (this.userData.preferredStreamExpressions?.length) {
const selector = new StreamSelector(context.toExpressionContext());
const streamToConditionIndex = new Map<string, number>();
// Go through each preferred filter condition, from highest to lowest priority.
for (
let i = 0;
i < this.userData.preferredStreamExpressions.length;
i++
) {
const item = this.userData.preferredStreamExpressions[i];
const { expression, enabled } = item;
if (!enabled) continue;
// From the streams that haven't been matched to a higher-priority condition yet...
const availableStreams = streams.filter(
(stream) => !streamToConditionIndex.has(stream.id)
);
// ...select the ones that match the current condition.
try {
const selectedStreams = await selector.select(
availableStreams,
expression
);
// And for each of those, record that this is the best condition they've matched so far.
for (const stream of selectedStreams) {
streamToConditionIndex.set(stream.id, i);
}
} catch (error) {
logger.error(
{
expression,
err: error instanceof Error ? error.message : String(error),
},
'failed to apply preferred stream expression'
);
}
}
// Now, apply the results to the original streams list.
for (const stream of streams) {
const conditionIndex = streamToConditionIndex.get(stream.id);
if (conditionIndex !== undefined) {
const expression =
this.userData.preferredStreamExpressions[conditionIndex].expression;
stream.streamExpressionMatched = {
index: conditionIndex,
name: extractNamesFromExpression(expression)?.[0],
};
}
}
}
return Date.now() - start;
}
}
export default StreamPrecomputer;