Spaces:
Paused
Paused
| import z from 'zod'; | |
| import { UserData } from '../db/schemas.js'; | |
| import { config } from '../config/index.js'; | |
| import { | |
| SyncManager, | |
| type SyncOverride, | |
| type FetchResult, | |
| parseSyncedUrl, | |
| } from './sync.js'; | |
| import { createLogger } from '../logging/logger.js'; | |
| const logger = createLogger('core'); | |
| /** | |
| * Schema for a regex pattern item fetched from a sync URL. | |
| */ | |
| const RegexPatternSchema = z.object({ | |
| name: z.string().optional(), | |
| pattern: z.string(), | |
| score: z.number().optional(), | |
| }); | |
| export type RegexPatternItem = z.infer<typeof RegexPatternSchema> & { | |
| name: string; | |
| }; | |
| /** | |
| * Manages regex pattern whitelisting, access control, and URL syncing. | |
| * | |
| * Access model: | |
| * - `REGEX_FILTER_ACCESS = 'all'` → anyone can use any regex / sync from any URL | |
| * - `REGEX_FILTER_ACCESS = 'trusted'` → trusted users can use any regex; others are limited to whitelisted patterns | |
| * - `REGEX_FILTER_ACCESS = 'none'` → no one can use regex (except whitelisted patterns) | |
| */ | |
| export class RegexAccess { | |
| private static _instance: SyncManager<RegexPatternItem>; | |
| private static _whitelistedPatterns: string[] = []; | |
| private static _description?: string; | |
| /** | |
| * Get or create the singleton SyncManager instance. | |
| */ | |
| private static get manager(): SyncManager<RegexPatternItem> { | |
| if (!this._instance) { | |
| this._whitelistedPatterns = config.userLimits.regex.patterns; | |
| this._description = | |
| config.userLimits.regex.patternsDescription ?? undefined; | |
| const configuredUrls = config.userLimits.regex.patternsUrls; | |
| const refreshInterval = config.userLimits.sync.refreshInterval; | |
| this._instance = new SyncManager<RegexPatternItem>({ | |
| cacheKey: 'regex-patterns', | |
| maxCacheSize: 100, | |
| refreshInterval, | |
| configuredUrls, | |
| itemSchema: RegexPatternSchema, | |
| itemKey: (item) => item.pattern, | |
| convertValue: (v) => ({ name: v, pattern: v }), | |
| }); | |
| } | |
| return this._instance; | |
| } | |
| /** | |
| * Initialise the regex access service. | |
| */ | |
| public static initialise(): Promise<void> { | |
| // Seed the manager with statically configured patterns | |
| if (this._whitelistedPatterns.length > 0 || !this._instance) { | |
| // Ensure manager is created | |
| const mgr = this.manager; | |
| if (this._whitelistedPatterns.length > 0) { | |
| mgr.addItems( | |
| this._whitelistedPatterns.map((p) => ({ name: p, pattern: p })) | |
| ); | |
| } | |
| } | |
| return this.manager.initialise(); | |
| } | |
| /** | |
| * Clean up resources. Safe to call before `initialise()`: the `manager` | |
| * getter would otherwise read from `config.userLimits.regex` and trip the | |
| * settings-store guard if shutdown runs before `initialiseConfig()` has | |
| * resolved (e.g. SIGTERM during startup). | |
| */ | |
| public static cleanup(): void { | |
| if (this._instance) this._instance.cleanup(); | |
| } | |
| /** | |
| * Add patterns to the accumulated whitelist. | |
| * Used by templates to register regex patterns that should be allowed | |
| * (any pattern in an instance-owner template is automatically trusted). | |
| */ | |
| public static addPatterns(patterns: string[]): void { | |
| this.manager.addItems(patterns.map((pattern) => ({ pattern, name: '' }))); | |
| } | |
| /** | |
| * Add URLs to the allowed list for regex pattern syncing. | |
| * URLs added this way are considered trusted and can be used for syncing. | |
| */ | |
| public static addAllowedUrls(urls: string[]): void { | |
| this.manager.addAllowedUrls(urls); | |
| Promise.all(urls.map((url) => this.getPatternsForUrl(url))) | |
| .then((results) => { | |
| const patterns = results.flat(); | |
| if (patterns.length > 0) { | |
| this.manager.addItems(patterns); | |
| } | |
| }) | |
| .catch((err) => | |
| logger.warn( | |
| `Failed to pre-fetch regex patterns from allowed URLs: ${err}` | |
| ) | |
| ); | |
| } | |
| /** | |
| * Get all allowed URLs for regex pattern syncing. | |
| */ | |
| public static getAllowedUrls(): string[] { | |
| return this.manager.allowedUrls; | |
| } | |
| /** | |
| * Check if a user is allowed to use regex filters. | |
| * If specific regexes are provided, checks if they're all in the whitelist. | |
| */ | |
| public static async isRegexAllowed( | |
| userData: UserData, | |
| regexes?: string[] | |
| ): Promise<boolean> { | |
| await this.initialise(); | |
| // If specific patterns are provided, check if all are whitelisted | |
| if (regexes && regexes.length > 0) { | |
| const whitelisted = this.manager.accumulatedKeys; | |
| const allWhitelisted = regexes.every((r) => whitelisted.has(r)); | |
| if (allWhitelisted) return true; | |
| } | |
| switch (config.userLimits.regex.access) { | |
| case 'trusted': | |
| return userData.trusted ?? false; | |
| case 'all': | |
| return true; | |
| default: | |
| return false; | |
| } | |
| } | |
| /** | |
| * Get the whitelisted regex patterns info (for status endpoint). | |
| */ | |
| public static async allowedRegexPatterns(): Promise<{ | |
| patterns: string[]; | |
| description?: string; | |
| urls: string[]; | |
| }> { | |
| await this.initialise(); | |
| return { | |
| patterns: [...this.manager.accumulatedKeys], | |
| description: this._description, | |
| urls: this.manager.allowedUrls, | |
| }; | |
| } | |
| /** | |
| * Validate sync URLs based on access level and user trust. | |
| * - `all` → any URL allowed | |
| * - `trusted` → trusted users can use any URL; others limited to configured URLs | |
| * - `none` → only configured URLs | |
| */ | |
| public static validateUrls(urls: string[], userData?: UserData): string[] { | |
| const access = config.userLimits.regex.access; | |
| const isUnrestricted = | |
| access === 'all' || (access === 'trusted' && userData?.trusted); | |
| if (isUnrestricted) return urls; | |
| return urls.filter((url) => this.manager.allowedUrls.includes(url)); | |
| } | |
| /** | |
| * Fetch patterns from a single URL (for direct access). | |
| */ | |
| public static async getPatternsForUrl( | |
| url: string | |
| ): Promise<RegexPatternItem[]> { | |
| return this.manager.fetchFromUrl(url); | |
| } | |
| /** | |
| * Resolve patterns from URLs with validation. | |
| */ | |
| public static async resolvePatterns( | |
| urls: string[] | undefined, | |
| userData?: UserData | |
| ): Promise<RegexPatternItem[]> { | |
| if (!urls?.length) return []; | |
| const validUrls = this.validateUrls(urls, userData); | |
| if (!validUrls.length) return []; | |
| // Track dynamic URLs | |
| const mgr = this.manager; | |
| for (const url of validUrls) { | |
| (mgr as any)._dynamicUrls ??= new Set(); | |
| } | |
| const allPatterns = await Promise.all( | |
| validUrls.map((url) => this.getPatternsForUrl(url)) | |
| ); | |
| const patterns = allPatterns.flat(); | |
| // Add fetched patterns to the accumulated whitelist | |
| if (patterns.length > 0) { | |
| mgr.addItems(patterns); | |
| } | |
| return patterns; | |
| } | |
| /** | |
| * Resolve patterns from URLs, returning per-URL results with errors. | |
| * Used by the API route to forward errors to the frontend. | |
| */ | |
| public static async resolvePatternsWithErrors( | |
| urls: string[] | undefined, | |
| userData?: UserData | |
| ): Promise<FetchResult<RegexPatternItem>[]> { | |
| if (!urls?.length) return []; | |
| const validUrls = new Set(this.validateUrls(urls, userData)); | |
| const results = await Promise.all( | |
| urls.map((url) => { | |
| if (!validUrls.has(url)) { | |
| return { | |
| url, | |
| items: [] as RegexPatternItem[], | |
| error: | |
| config.userLimits.regex.access === 'none' | |
| ? 'Regex sync is disabled on this instance.' | |
| : config.userLimits.regex.access === 'trusted' && | |
| !userData?.trusted | |
| ? 'This URL is not in the allowed list. Contact the instance owner to whitelist it, or ask to be marked as a trusted user.' | |
| : 'This URL is not allowed by the server configuration.', | |
| } satisfies FetchResult<RegexPatternItem>; | |
| } | |
| return this.manager.fetchFromUrlWithError(url); | |
| }) | |
| ); | |
| // Add successful patterns to the whitelist | |
| const allPatterns = results.flatMap((r) => r.items); | |
| if (allPatterns.length > 0) { | |
| this.manager.addItems(allPatterns); | |
| } | |
| return results; | |
| } | |
| /** | |
| * Sync regex patterns from URLs into the user's existing patterns. | |
| * This is the main method called by the middleware. | |
| * | |
| * Resolves `<SYNCED: url>` inline placeholders in-place; unplaced URLs | |
| * are appended at the end. Dangling placeholders are stripped. | |
| */ | |
| public static async syncRegexPatterns<U>( | |
| urls: string[] | undefined, | |
| existing: U[], | |
| userData: UserData, | |
| transform: (item: RegexPatternItem) => U, | |
| getField: (item: U) => string | |
| ): Promise<U[]> { | |
| const validUrls = urls?.length ? this.validateUrls(urls, userData) : []; | |
| if (validUrls.length === 0) { | |
| const cleaned = existing.filter( | |
| (item) => !parseSyncedUrl(getField(item)) | |
| ); | |
| return cleaned.length === existing.length ? existing : cleaned; | |
| } | |
| const validUrlSet = new Set(validUrls); | |
| const urlPatternMap = new Map<string, RegexPatternItem[]>(); | |
| await Promise.all( | |
| validUrls.map(async (url) => { | |
| const patterns = await this.getPatternsForUrl(url); | |
| urlPatternMap.set(url, patterns); | |
| }) | |
| ); | |
| const allPatterns = [...urlPatternMap.values()].flat(); | |
| if (allPatterns.length > 0) { | |
| this.manager.addItems(allPatterns); | |
| } | |
| const overrides: SyncOverride[] = userData.regexOverrides || []; | |
| const result: U[] = []; | |
| const resolvedInlineUrls = new Set<string>(); | |
| const pushPatterns = (patterns: RegexPatternItem[]) => { | |
| for (const regex of patterns) { | |
| const override = overrides.find( | |
| (o) => | |
| o.pattern === regex.pattern || | |
| (regex.name && o.originalName === regex.name) | |
| ); | |
| if (override?.disabled) continue; | |
| result.push( | |
| transform( | |
| override | |
| ? { | |
| ...regex, | |
| name: override.name ?? regex.name, | |
| score: | |
| override.score !== undefined ? override.score : regex.score, | |
| } | |
| : regex | |
| ) | |
| ); | |
| } | |
| }; | |
| for (const item of existing) { | |
| const placeholderUrl = parseSyncedUrl(getField(item)); | |
| if (placeholderUrl) { | |
| if (validUrlSet.has(placeholderUrl)) { | |
| resolvedInlineUrls.add(placeholderUrl); | |
| pushPatterns(urlPatternMap.get(placeholderUrl) ?? []); | |
| } | |
| continue; | |
| } | |
| result.push(item); | |
| } | |
| for (const url of validUrls) { | |
| if (resolvedInlineUrls.has(url)) continue; | |
| pushPatterns(urlPatternMap.get(url) ?? []); | |
| } | |
| return result; | |
| } | |
| /** | |
| * Helper method to resolve all synced regex patterns from URLs for temporary validation. | |
| * Returns patterns without modifying the userData config. | |
| * Used by config validation to merge synced patterns temporarily. | |
| */ | |
| public static async resolveSyncedRegexesForValidation( | |
| userData: UserData | |
| ): Promise<{ | |
| included: string[]; | |
| excluded: string[]; | |
| required: string[]; | |
| preferred: { name: string; pattern: string; score?: number }[]; | |
| ranked: { name?: string; pattern: string; score: number }[]; | |
| }> { | |
| try { | |
| const [included, excluded, required, preferred, ranked] = | |
| await Promise.all([ | |
| this.syncRegexPatterns( | |
| userData.syncedIncludedRegexUrls, | |
| [], | |
| userData, | |
| (regex) => regex.pattern, | |
| (pattern) => pattern | |
| ), | |
| this.syncRegexPatterns( | |
| userData.syncedExcludedRegexUrls, | |
| [], | |
| userData, | |
| (regex) => regex.pattern, | |
| (pattern) => pattern | |
| ), | |
| this.syncRegexPatterns( | |
| userData.syncedRequiredRegexUrls, | |
| [], | |
| userData, | |
| (regex) => regex.pattern, | |
| (pattern) => pattern | |
| ), | |
| this.syncRegexPatterns( | |
| userData.syncedPreferredRegexUrls, | |
| [], | |
| userData, | |
| (regex) => regex, | |
| (regex) => regex.pattern | |
| ), | |
| this.syncRegexPatterns( | |
| userData.syncedRankedRegexUrls, | |
| [], | |
| userData, | |
| (regex) => ({ | |
| pattern: regex.pattern, | |
| name: regex.name, | |
| score: regex.score || 0, | |
| }), | |
| (item) => item.pattern | |
| ), | |
| ]); | |
| return { included, excluded, required, preferred, ranked }; | |
| } catch (err) { | |
| throw new Error( | |
| err instanceof Error | |
| ? `Failed to resolve one or more synced regex patterns: ${err.message}` | |
| : 'Failed to resolve one or more synced regex patterns' | |
| ); | |
| } | |
| } | |
| } | |