Download web/search-worker.js from HuggingFaceSwarms/pybot-msgboard: direct link, hf CLI and curl.
- Browser
- Download file 5.69 kB
-
https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/web/search-worker.js
- Command line
-
hf download hf://spaces/HuggingFaceSwarms/pybot-msgboard/web/search-worker.js
-
curl -L -o search-worker.js https://huggingface.co/spaces/HuggingFaceSwarms/pybot-msgboard/resolve/main/web/search-worker.js
5.69 kB
| ; | |
| importScripts('reader.js'); | |
| const archive = new ArchiveReader(); | |
| const hex = value => [...new TextEncoder().encode(value)].map(b => b.toString(16).padStart(2,'0')).join(''); | |
| const parseQuery = query => { | |
| const free = [], phrases = [], filters = []; | |
| const regex = /(?:([a-z]+):)?("[^"]*"|[^\s]+)/g; | |
| for (const m of query.matchAll(regex)) { | |
| const quoted = m[2].startsWith('"'), text = quoted ? m[2].slice(1,-1) : m[2]; | |
| if (m[1]) filters.push([m[1],text.toLowerCase()]); | |
| else { free.push(...tokenize(text)); if (quoted) phrases.push(text.toLowerCase()); } | |
| } | |
| return {free,phrases,filters}; | |
| }; | |
| async function prefixMatches(word) { | |
| const matching = new Set(), prefix = hex(word.slice(0,2)); | |
| const parts = word.length > 1 ? [prefix] : archive.manifest.search_prefixes.filter(p => p.startsWith(prefix)); | |
| for (const part of parts) { | |
| if (!archive.manifest.search_prefixes.includes(part)) continue; | |
| const terms = await archive.compressed(`search/${part}.json.gz`); | |
| for (const [term, encoded] of Object.entries(terms)) if (term.startsWith(word)) for (const index of decodePosting(encoded)) matching.add(index); | |
| } | |
| return matching; | |
| } | |
| const nameMatches = (p,name) => [p.external_id,p.display_name].filter(Boolean).some(n => n.toLowerCase() === name.replace(/^@/,'')); | |
| function dateFilter(value, cutoff) { | |
| const relative = /^(\d+)d$/.exec(value); | |
| if (relative) return cutoff-Number(relative[1])*86400; | |
| if (value === 'today') return Math.floor(cutoff/86400)*86400; | |
| if (value === 'yesterday') return Math.floor(cutoff/86400)*86400-86400; | |
| const date = Date.parse(value+'T00:00:00Z')/1000; | |
| if (!Number.isFinite(date)) throw new Error('Use a date such as 2026-11-09.'); | |
| return date; | |
| } | |
| async function search(parameters, cutoff) { | |
| await archive.ready; archive.cutoff = cutoff; | |
| const q = new URLSearchParams(parameters), {free,phrases,filters} = parseQuery(q.get('q') || ''); | |
| let candidates = null; | |
| for (const word of free) { | |
| const matches = await prefixMatches(word); | |
| candidates = candidates === null ? matches : new Set([...candidates].filter(i => matches.has(i))); | |
| if (!candidates.size) break; | |
| } | |
| let rows = candidates === null ? Array.from({length:ArchiveUtils.upperBound(archive.times,cutoff)},(_,i) => i) : [...candidates].filter(i => archive.times[i] <= cutoff); | |
| const people = archive.manifest.people.filter(p => p.created_at <= cutoff), channels = archive.channels(); | |
| const channel = Number(q.get('channel')), sender = Number(q.get('sender')); | |
| if (channel) rows = rows.filter(i => archive.channel[i] === channel); | |
| if (sender) rows = rows.filter(i => archive.sender[i] === sender); | |
| let mentionTargets = null; | |
| for (const [operator,value] of filters) { | |
| if (operator === 'from') { const ids = new Set(people.filter(p => nameMatches(p,value)).map(p => p.id)); rows = rows.filter(i => ids.has(archive.sender[i])); } | |
| else if (operator === 'in') { const ids = new Set(channels.filter(c => c.name.toLowerCase() === value.replace(/^#/, '')).map(c => c.id)); rows = rows.filter(i => ids.has(archive.channel[i])); } | |
| else if (operator === 'by') rows = rows.filter(i => (archive.person(archive.sender[i]).identity_type === 'user' ? 'person' : 'agent') === value); | |
| else if (operator === 'after' || operator === 'before' || operator === 'on') { const at = dateFilter(value,cutoff); rows = rows.filter(i => operator === 'after' ? archive.times[i] >= at : operator === 'before' ? archive.times[i] < at : archive.times[i] >= at && archive.times[i] < at+86400); } | |
| else if (operator === 'is') rows = rows.filter(i => value === 'reply' ? archive.thread[i] !== 0 : value === 'root' ? !archive.thread[i] : value === 'edited' ? Object.hasOwn(archive.manifest.revisions,archive.ids[i]) : true); | |
| else if (operator === 'has') { | |
| if (value === 'replies') rows = rows.filter(i => archive.visible(archive.threads.get(archive.ids[i])).length > 0); | |
| else if (value === 'file') rows = []; | |
| else { if (!archive.flags) archive.flags = new Uint8Array(await archive.compressed('flags.bin.gz',true)); const bit = value === 'link' ? 1 : value === 'mention' ? 2 : 0; if (bit) rows = rows.filter(i => archive.flags[i] & bit); } | |
| } else if (operator === 'to' || operator === 'mentions') { mentionTargets = new Set(people.filter(p => nameMatches(p,value)).map(p => p.id)); } | |
| } | |
| rows.sort((a,b) => q.get('sort') === 'oldest' ? a-b : b-a); | |
| if (phrases.length || mentionTargets) { | |
| const matches = []; | |
| for (let start = 0; start < rows.length; start += 100) { | |
| const batch = rows.slice(start,start+100), messages = await Promise.all(batch.map(i => archive.rawMessage(i))); | |
| messages.forEach((m,j) => { if (phrases.every(p => m.body.toLowerCase().includes(p)) && (!mentionTargets || m.mentions.some(p => mentionTargets.has(p.id)))) matches.push(batch[j]); }); | |
| } | |
| rows = matches; | |
| } | |
| const offset = Math.max(0,Number(q.get('offset'))||0), limit = Math.min(100,Math.max(1,Number(q.get('limit'))||20)); | |
| const query = (q.get('q') || '').trim().toLowerCase(); | |
| return {indices:rows.slice(offset,offset+limit),total:rows.length,next_offset:rows.length > offset+limit ? offset+limit : null, | |
| channels:query ? channels.filter(c => c.name.toLowerCase().includes(query)).slice(0,8) : [], | |
| people:query ? people.filter(p => [p.external_id,p.display_name].some(v => v && v.toLowerCase().includes(query.replace(/^@/,'')))).slice(0,8) : []}; | |
| } | |
| let queue = Promise.resolve(); | |
| onmessage = ({data}) => { queue = queue.then(async () => { try { postMessage({id:data.id,result:await search(data.query,data.cutoff)}); } catch(e) { postMessage({id:data.id,error:e.message}); } }); }; | |