aiostreams / packages /core /src /utils /sel-access.ts
f4b404's picture
Upload folder using huggingface_hub
f55c62a verified
Raw
History Blame Contribute Delete
11.9 kB
import z from 'zod';
import { UserData } from '../db/schemas.js';
import { config } from '../config/index.js';
import {
SyncManager,
type SyncOverride,
type FetchResult,
parseSyncedUrl,
} from './sync.js';
import { extractNamesFromExpression } from '../parser/streamExpression.js';
import { createLogger } from '../logging/logger.js';
const logger = createLogger('core');
/**
* Schema for a stream expression item fetched from a sync URL.
*/
const StreamExpressionSchema = z.object({
expression: z.string().min(1),
name: z.string().optional(),
score: z.number().optional(),
enabled: z.boolean().optional(),
});
export type StreamExpressionItem = z.infer<typeof StreamExpressionSchema>;
/**
* Manages stream expression (SEL) URL syncing and access control.
*
* Access model (controls which sync URLs can be used, NOT which expressions can be entered):
* - `SEL_SYNC_ACCESS = 'all'` → anyone can sync from any URL
* - `SEL_SYNC_ACCESS = 'trusted'` → trusted users can sync from any URL;
* others can only sync from WHITELISTED_SEL_URLS
*
* Users can always enter any SEL expression locally - access control only applies to sync URLs.
*/
export class SelAccess {
private static _instance: SyncManager<StreamExpressionItem>;
/**
* Get or create the singleton SyncManager instance.
*/
private static get manager(): SyncManager<StreamExpressionItem> {
if (!this._instance) {
const configuredUrls = config.userLimits.sel.urls;
const refreshInterval = config.userLimits.sync.refreshInterval;
this._instance = new SyncManager<StreamExpressionItem>({
cacheKey: 'sel-expressions',
maxCacheSize: 100,
refreshInterval,
configuredUrls,
itemSchema: StreamExpressionSchema,
itemKey: (item) => item.expression,
convertValue: (v) => ({ expression: v }),
});
}
return this._instance;
}
/**
* Initialise the SEL access service.
*/
public static initialise(): Promise<void> {
return this.manager.initialise();
}
/**
* Clean up resources. Safe to call before `initialise()`: the `manager`
* getter would otherwise read from `config.userLimits.sel` and trip the
* settings-store guard if shutdown runs before `initialiseConfig()` has
* resolved (e.g. SIGTERM during startup).
*/
public static cleanup(): void {
if (this._instance) this._instance.cleanup();
}
/**
* Validate sync URLs based on access level and user trust.
* - `all` → any URL allowed
* - `trusted` → trusted users can use any URL; others limited to whitelisted SEL URLs
*/
public static validateUrls(urls: string[], userData?: UserData): string[] {
const access = config.userLimits.sel.access;
const isUnrestricted =
access === 'all' || (access === 'trusted' && userData?.trusted);
if (isUnrestricted) return urls;
// Non-trusted users can only use whitelisted SEL URLs
return urls.filter((url) => this.manager.allowedUrls.includes(url));
}
/**
* Fetch expressions from a single URL.
*/
public static async getExpressionsForUrl(
url: string
): Promise<StreamExpressionItem[]> {
return this.manager.fetchFromUrl(url);
}
/**
* Resolve expressions from URLs with validation.
*/
public static async resolveExpressions(
urls: string[] | undefined,
userData?: UserData
): Promise<StreamExpressionItem[]> {
if (!urls?.length) return [];
const validUrls = this.validateUrls(urls, userData);
if (!validUrls.length) return [];
const results = await Promise.all(
validUrls.map((url) => this.getExpressionsForUrl(url))
);
return results.flat();
}
/**
* Resolve expressions from URLs, returning per-URL results with errors.
* Used by the API route to forward errors to the frontend.
*/
public static async resolveExpressionsWithErrors(
urls: string[] | undefined,
userData?: UserData
): Promise<FetchResult<StreamExpressionItem>[]> {
if (!urls?.length) return [];
const validUrls = new Set(this.validateUrls(urls, userData));
const results = await Promise.all(
urls.map((url) => {
if (!validUrls.has(url)) {
return {
url,
items: [] as StreamExpressionItem[],
error:
config.userLimits.sel.access === 'trusted' && !userData?.trusted
? 'This URL is not in the allowed list. Contact the instance owner to whitelist it, or ask to be marked as a trusted user.'
: 'This URL is not allowed by the server configuration.',
} satisfies FetchResult<StreamExpressionItem>;
}
return this.manager.fetchFromUrlWithError(url);
})
);
return results;
}
/**
* Sync stream expressions from URLs into the user's existing expressions.
* This is the main method called by the middleware.
*
* Override logic for SEL:
* - **disabled**: skip the expression entirely
* - **score override (ranked only)**: override the score value
*
* Override matching:
* - By exact expression string match (`override.expression === item.expression`)
* - By extracted names match: names extracted from expression comments vs `override.exprNames`
*
* Resolves `<SYNCED: url>` inline placeholders in-place; unplaced URLs
* are appended at the end. Dangling placeholders are stripped.
*/
public static async syncStreamExpressions<U>(
urls: string[] | undefined,
existing: U[],
userData: UserData,
transform: (item: StreamExpressionItem) => U,
getField: (item: U) => string
): Promise<U[]> {
const validUrls = urls?.length ? this.validateUrls(urls, userData) : [];
if (validUrls.length === 0) {
const cleaned = existing.filter(
(item) => !parseSyncedUrl(getField(item))
);
return cleaned.length === existing.length ? existing : cleaned;
}
const validUrlSet = new Set(validUrls);
const urlExprMap = new Map<string, StreamExpressionItem[]>();
await Promise.all(
validUrls.map(async (url) => {
const exprs = await this.getExpressionsForUrl(url);
urlExprMap.set(url, exprs);
})
);
const overrides: SyncOverride[] = userData.selOverrides || [];
const result: U[] = [];
const resolvedInlineUrls = new Set<string>();
const pushExpressions = (expressions: StreamExpressionItem[]) => {
for (const expr of expressions) {
const override = this._findSelOverride(expr, overrides);
if (override?.disabled) continue;
const overriddenExpr = override
? this._applySelOverride(expr, override)
: expr;
result.push(transform(overriddenExpr));
}
};
for (const item of existing) {
const placeholderUrl = parseSyncedUrl(getField(item));
if (placeholderUrl) {
if (validUrlSet.has(placeholderUrl)) {
resolvedInlineUrls.add(placeholderUrl);
pushExpressions(urlExprMap.get(placeholderUrl) ?? []);
}
continue;
}
result.push(item);
}
for (const url of validUrls) {
if (resolvedInlineUrls.has(url)) continue;
pushExpressions(urlExprMap.get(url) ?? []);
}
return result;
}
/**
* Add URLs to the allowed list for SEL syncing.
* URLs added this way are considered trusted and can be used for syncing.
*/
public static addAllowedUrls(urls: string[]): void {
this.manager.addAllowedUrls(urls);
}
/**
* Get all allowed URLs for SEL syncing.
*/
public static getAllowedUrls(): string[] {
return this.manager.allowedUrls;
}
/**
* Find a matching SEL override for an expression.
* Matches by exact expression string or by comparing extracted names
* from the expression against the override's stored `exprNames` array.
*/
private static _findSelOverride(
expr: StreamExpressionItem,
overrides: SyncOverride[]
): SyncOverride | undefined {
if (!overrides.length) return undefined;
return overrides.find((o) => {
// Match by exact expression
if (o.expression && o.expression === expr.expression) return true;
// Match by extracted names vs stored exprNames
if (o.exprNames && o.exprNames.length > 0) {
const names = extractNamesFromExpression(expr.expression, false);
const matches = (list?: string[]) =>
!!list &&
list.length === o.exprNames!.length &&
list.every((n, i) => n === o.exprNames![i]);
if (matches(names)) {
return true;
}
}
return false;
});
}
/**
* Apply an SEL override to an expression item.
* Supports score overrides and enabled state overrides.
*/
private static _applySelOverride(
expr: StreamExpressionItem,
override: SyncOverride
): StreamExpressionItem {
const result = { ...expr };
if (override.score !== undefined) {
result.score = override.score;
}
// If the user explicitly set disabled=false, override enabled to true
if (override.disabled === false && expr.enabled === false) {
result.enabled = true;
}
return result;
}
/**
* Helper method to resolve all synced stream expressions from URLs for temporary validation.
* Returns expressions without modifying the userData config.
* Used by config validation to merge synced expressions temporarily.
*/
public static async resolveSyncedExpressionsForValidation(
userData: UserData
): Promise<{
included: { expression: string; enabled: boolean }[];
excluded: { expression: string; enabled: boolean }[];
required: { expression: string; enabled: boolean }[];
preferred: { expression: string; enabled: boolean }[];
ranked: { expression: string; score: number; enabled: boolean }[];
}> {
try {
const [included, excluded, required, preferred, ranked] =
await Promise.all([
this.syncStreamExpressions(
userData.syncedIncludedStreamExpressionUrls,
[],
userData,
(item) => ({
expression: item.expression,
enabled: item.enabled ?? true,
}),
(item) => item.expression
),
this.syncStreamExpressions(
userData.syncedExcludedStreamExpressionUrls,
[],
userData,
(item) => ({
expression: item.expression,
enabled: item.enabled ?? true,
}),
(item) => item.expression
),
this.syncStreamExpressions(
userData.syncedRequiredStreamExpressionUrls,
[],
userData,
(item) => ({
expression: item.expression,
enabled: item.enabled ?? true,
}),
(item) => item.expression
),
this.syncStreamExpressions(
userData.syncedPreferredStreamExpressionUrls,
[],
userData,
(item) => ({
expression: item.expression,
enabled: item.enabled ?? true,
}),
(item) => item.expression
),
this.syncStreamExpressions(
userData.syncedRankedStreamExpressionUrls,
[],
userData,
(item) => ({
expression: item.expression,
score: item.score || 0,
enabled: item.enabled ?? true,
}),
(item) => item.expression
),
]);
return { included, excluded, required, preferred, ranked };
} catch (err) {
throw new Error(
err instanceof Error
? `Failed to resolve one or more synced stream expressions: ${err.message}`
: 'Failed to resolve one or more synced stream expressions'
);
}
}
}