Add missing scripts folder

scripts/generate-sbom.sh

@@ -0,0 +1,194 @@
+#!/usr/bin/env bash
+#
+# generate-sbom.sh — Generate a clean CycloneDX SBOM using Syft
+#
+# Produces a single SBOM from resolved manifests only — no directory scanning,
+# no venv pollution, no local state. Works identically locally and in CI.
+#
+# How it works:
+#   1. Python: uv pip compile resolves all transitive deps from requirements.txt
+#   2. JavaScript: package-lock.json already contains the full resolved tree
+#   3. Syft scans these resolved files, not the filesystem
+#
+# Usage:
+#   ./scripts/generate-sbom.sh              # generate sbom.cdx.json from manifests
+#   ./scripts/generate-sbom.sh docker       # generate from Docker image (best license coverage)
+#   ./scripts/generate-sbom.sh docker IMG   # generate from a specific image
+#   ./scripts/generate-sbom.sh validate     # validate existing SBOM
+#
+# Requirements:
+#   - syft (brew install syft)
+#   - uv  (brew install uv)
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+ROOT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)"
+
+RED='\033[0;31m'
+GREEN='\033[0;32m'
+DIM='\033[2m'
+BOLD='\033[1m'
+RESET='\033[0m'
+
+info()  { echo -e "${BOLD}${GREEN}▸${RESET} $1"; }
+warn()  { echo -e "${BOLD}${RED}▸${RESET} $1"; }
+dim()   { echo -e "${DIM}  $1${RESET}"; }
+
+OUTPUT="$ROOT_DIR/sbom.cdx.json"
+
+check_deps() {
+    local missing=()
+    command -v syft &>/dev/null || missing+=("syft")
+    command -v uv &>/dev/null   || missing+=("uv")
+    if [[ ${#missing[@]} -gt 0 ]]; then
+        warn "Missing: ${missing[*]}. Install with: brew install ${missing[*]}"
+        exit 1
+    fi
+    dim "Using $(syft --version), $(uv --version)"
+}
+
+generate() {
+    info "Generating SBOM from resolved manifests..."
+    check_deps
+
+    local VERSION
+    VERSION="$(python3 -c "import json; print(json.load(open('$ROOT_DIR/package.json'))['version'])")"
+
+    local WORK_DIR
+    WORK_DIR="$(mktemp -d)"
+    trap 'rm -rf "$WORK_DIR"' RETURN
+
+    # --- Python: resolve all transitive deps without installing ---
+    dim "Resolving Python transitive deps (uv pip compile)..."
+    uv pip compile "$ROOT_DIR/backend/requirements.txt" \
+        --python-version 3.11 \
+        --quiet \
+        > "$WORK_DIR/requirements-resolved.txt" 2>/dev/null
+
+    # --- JavaScript: package-lock.json is already fully resolved ---
+    if [[ -f "$ROOT_DIR/package-lock.json" ]]; then
+        cp "$ROOT_DIR/package-lock.json" "$WORK_DIR/package-lock.json"
+        # Syft needs package.json alongside the lockfile
+        cp "$ROOT_DIR/package.json" "$WORK_DIR/package.json"
+    else
+        warn "package-lock.json not found — JS deps will be skipped"
+    fi
+
+    # --- Scan only the resolved files ---
+    dim "Scanning resolved manifests with Syft..."
+    syft scan "dir:$WORK_DIR" \
+        --output "cyclonedx-json=$OUTPUT" \
+        --source-name open-webui \
+        --source-version "$VERSION" \
+        --quiet
+
+    # Print summary
+    python3 -c "
+import json
+with open('$OUTPUT') as f:
+    data = json.load(f)
+comps = data.get('components', [])
+py = [c for c in comps if 'pypi' in c.get('purl', '')]
+js = [c for c in comps if 'npm' in c.get('purl', '')]
+with_lic = sum(1 for c in comps if c.get('licenses'))
+print(f'  {len(comps)} total ({len(py)} Python, {len(js)} JavaScript)')
+print(f'  {with_lic}/{len(comps)} with license info')
+print(f'  Serial: {data.get(\"serialNumber\", \"none\")}')
+print(f'  Timestamp: {data.get(\"metadata\", {}).get(\"timestamp\", \"none\")}')
+"
+
+    info "SBOM written → sbom.cdx.json"
+}
+
+generate_docker() {
+    local IMAGE="${1:-ghcr.io/open-webui/open-webui:latest}"
+    info "Generating SBOM from Docker image: $IMAGE"
+
+    if ! command -v syft &>/dev/null; then
+        warn "syft is not installed. Install with: brew install syft"
+        exit 1
+    fi
+
+    dim "Pulling and scanning image..."
+    syft scan "docker:$IMAGE" \
+        --output "cyclonedx-json=$OUTPUT" \
+        --quiet
+
+    python3 -c "
+import json
+with open('$OUTPUT') as f:
+    data = json.load(f)
+comps = data.get('components', [])
+with_lic = sum(1 for c in comps if c.get('licenses'))
+print(f'  {len(comps)} total components')
+print(f'  {with_lic}/{len(comps)} with license info ({round(with_lic/max(len(comps),1)*100)}%)')
+"
+
+    info "SBOM written → sbom.cdx.json"
+}
+
+validate() {
+    info "Validating SBOM..."
+
+    python3 -c "
+import json, sys
+
+try:
+    with open('$OUTPUT') as f:
+        data = json.load(f)
+except FileNotFoundError:
+    print('  ✗ sbom.cdx.json not found — run ./scripts/generate-sbom.sh first')
+    sys.exit(1)
+
+issues = []
+if data.get('bomFormat') != 'CycloneDX':
+    issues.append('Not CycloneDX format')
+if not data.get('specVersion'):
+    issues.append('Missing specVersion')
+if not data.get('serialNumber'):
+    issues.append('Missing serial number')
+
+components = data.get('components', [])
+
+# Check for phantom local packages
+phantoms = []
+for c in components:
+    for ref in c.get('externalReferences', []):
+        url = ref.get('url', '')
+        if 'file://' in url and '/Users/' in url:
+            phantoms.append(c['name'])
+if phantoms:
+    issues.append(f'Phantom local packages: {phantoms}')
+
+with_lic = sum(1 for c in components if c.get('licenses'))
+lic_pct = round(with_lic / max(len(components), 1) * 100)
+
+if issues:
+    print(f'  ✗ {len(components)} components, {lic_pct}% licensed')
+    for i in issues:
+        print(f'    ✗ {i}')
+    sys.exit(1)
+else:
+    print(f'  ✓ {len(components)} components, {lic_pct}% licensed — PASS')
+"
+}
+
+# --- Main ---
+cd "$ROOT_DIR"
+TARGET="${1:-generate}"
+
+case "$TARGET" in
+    generate) generate ;;
+    docker)   generate_docker "${2:-}" ;;
+    validate) validate ;;
+    *)
+        warn "Unknown target: $TARGET"
+        echo "Usage: $0 [generate|docker [IMAGE]|validate]"
+        exit 1
+        ;;
+esac
+
+echo ""
+info "Done."

scripts/prepare-pyodide.js

@@ -0,0 +1,201 @@
+const packages = [
+	'micropip',
+	'packaging',
+	'requests',
+	'beautifulsoup4',
+	'numpy',
+	'pandas',
+	'matplotlib',
+	'scikit-learn',
+	'scipy',
+	'regex',
+	'sympy',
+	'tiktoken',
+	'seaborn',
+	'pytz',
+	'black',
+	'openai',
+	'openpyxl'
+];
+
+// Pure-Python packages whose wheels must be downloaded from PyPI and saved into
+// static/pyodide/ so that the browser can install them offline via micropip.
+// Packages already provided by the Pyodide distribution (click, platformdirs,
+// typing_extensions, etc.) do NOT need to be listed here.
+const pypiPackages = ['black', 'pathspec', 'mypy_extensions', 'pytokens'];
+
+import { loadPyodide } from 'pyodide';
+import { setGlobalDispatcher, ProxyAgent } from 'undici';
+import { writeFile, readFile, copyFile, readdir, rmdir, access } from 'fs/promises';
+
+/**
+ * Loading network proxy configurations from the environment variables.
+ * And the proxy config with lowercase name has the highest priority to use.
+ */
+function initNetworkProxyFromEnv() {
+	// we assume all subsequent requests in this script are HTTPS:
+	// https://cdn.jsdelivr.net
+	// https://pypi.org
+	// https://files.pythonhosted.org
+	const allProxy = process.env.all_proxy || process.env.ALL_PROXY;
+	const httpsProxy = process.env.https_proxy || process.env.HTTPS_PROXY;
+	const httpProxy = process.env.http_proxy || process.env.HTTP_PROXY;
+	const preferedProxy = httpsProxy || allProxy || httpProxy;
+	/**
+	 * use only http(s) proxy because socks5 proxy is not supported currently:
+	 * @see https://github.com/nodejs/undici/issues/2224
+	 */
+	if (!preferedProxy || !preferedProxy.startsWith('http')) return;
+	let preferedProxyURL;
+	try {
+		preferedProxyURL = new URL(preferedProxy).toString();
+	} catch {
+		console.warn(`Invalid network proxy URL: "${preferedProxy}"`);
+		return;
+	}
+	const dispatcher = new ProxyAgent({ uri: preferedProxyURL });
+	setGlobalDispatcher(dispatcher);
+	console.log(`Initialized network proxy "${preferedProxy}" from env`);
+}
+
+async function downloadPackages() {
+	console.log('Setting up pyodide + micropip');
+
+	let pyodide;
+	try {
+		pyodide = await loadPyodide({
+			packageCacheDir: 'static/pyodide'
+		});
+	} catch (err) {
+		console.error('Failed to load Pyodide:', err);
+		return;
+	}
+
+	const packageJson = JSON.parse(await readFile('package.json'));
+	const pyodideVersion = packageJson.dependencies.pyodide.replace('^', '');
+
+	try {
+		const pyodidePackageJson = JSON.parse(await readFile('static/pyodide/package.json'));
+		const pyodidePackageVersion = pyodidePackageJson.version.replace('^', '');
+
+		if (pyodideVersion !== pyodidePackageVersion) {
+			console.log('Pyodide version mismatch, removing static/pyodide directory');
+			await rmdir('static/pyodide', { recursive: true });
+		}
+	} catch (err) {
+		console.log('Pyodide package not found, proceeding with download.', err);
+	}
+
+	try {
+		console.log('Loading micropip package');
+		await pyodide.loadPackage('micropip');
+
+		const micropip = pyodide.pyimport('micropip');
+		console.log('Downloading Pyodide packages:', packages);
+
+		try {
+			for (const pkg of packages) {
+				console.log(`Installing package: ${pkg}`);
+				await micropip.install(pkg);
+			}
+		} catch (err) {
+			console.error('Package installation failed:', err);
+			return;
+		}
+
+		console.log('Pyodide packages downloaded, freezing into lock file');
+
+		try {
+			const lockFile = await micropip.freeze();
+			await writeFile('static/pyodide/pyodide-lock.json', lockFile);
+		} catch (err) {
+			console.error('Failed to write lock file:', err);
+		}
+	} catch (err) {
+		console.error('Failed to load or install micropip:', err);
+	}
+}
+
+async function copyPyodide() {
+	console.log('Copying Pyodide files into static directory');
+	// Copy all files from node_modules/pyodide to static/pyodide
+	for await (const entry of await readdir('node_modules/pyodide')) {
+		await copyFile(`node_modules/pyodide/${entry}`, `static/pyodide/${entry}`);
+	}
+}
+
+/**
+ * Download pure-Python wheels from PyPI and save them into static/pyodide/.
+ * Also injects entries into pyodide-lock.json so that micropip resolves these
+ * packages from the local server instead of fetching them from the internet.
+ */
+async function downloadPyPIWheels() {
+	const lockPath = 'static/pyodide/pyodide-lock.json';
+	let lockData;
+	try {
+		lockData = JSON.parse(await readFile(lockPath, 'utf-8'));
+	} catch {
+		console.warn('Could not read pyodide-lock.json, skipping PyPI wheel download');
+		return;
+	}
+
+	for (const pkg of pypiPackages) {
+		console.log(`Fetching PyPI metadata for: ${pkg}`);
+		const res = await fetch(`https://pypi.org/pypi/${pkg}/json`);
+		if (!res.ok) {
+			console.error(`Failed to fetch PyPI metadata for ${pkg}: ${res.status}`);
+			continue;
+		}
+		const meta = await res.json();
+		const version = meta.info.version;
+		const files = meta.urls || [];
+		// Find the pure-Python wheel (py3-none-any)
+		const wheel = files.find(
+			(f) => f.filename.endsWith('.whl') && f.filename.includes('py3-none-any')
+		);
+		if (!wheel) {
+			console.warn(`No pure-Python wheel found for ${pkg}==${version}, skipping`);
+			continue;
+		}
+		const dest = `static/pyodide/${wheel.filename}`;
+		// Download wheel if not already present
+		try {
+			await access(dest);
+			console.log(`  Already exists: ${wheel.filename}`);
+		} catch {
+			console.log(`  Downloading: ${wheel.filename}`);
+			const wheelRes = await fetch(wheel.url);
+			if (!wheelRes.ok) {
+				console.error(`  Failed to download ${wheel.filename}: ${wheelRes.status}`);
+				continue;
+			}
+			const buffer = Buffer.from(await wheelRes.arrayBuffer());
+			await writeFile(dest, buffer);
+			console.log(`  Saved: ${dest} (${buffer.length} bytes)`);
+		}
+
+		// Inject into pyodide-lock.json so micropip resolves locally
+		const normalizedName = pkg.replace(/-/g, '_');
+		if (!lockData.packages[normalizedName]) {
+			lockData.packages[normalizedName] = {
+				name: normalizedName,
+				version: version,
+				file_name: wheel.filename,
+				install_dir: 'site',
+				sha256: wheel.digests?.sha256 || '',
+				package_type: 'package',
+				imports: [normalizedName],
+				depends: []
+			};
+			console.log(`  Added ${normalizedName}==${version} to pyodide-lock.json`);
+		}
+	}
+
+	await writeFile(lockPath, JSON.stringify(lockData, null, 2));
+	console.log('Updated pyodide-lock.json with PyPI packages');
+}
+
+initNetworkProxyFromEnv();
+await downloadPackages();
+await copyPyodide();
+await downloadPyPIWheels();