File size: 2,961 Bytes
559972f | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 | // The same byte BPE and Unicode pre-splitting as tokenizer.py.
const pattern = /'(?:[sSdDmMtT]|[lL][lL]|[vV][eE]|[rR][eE])|[^\r\n\p{L}\p{N}]?\p{L}+|\p{N}{1,3}| ?[^\p{White_Space}\p{L}\p{N}]+[\r\n]*|\p{White_Space}*[\r\n]|\p{White_Space}+(?![^\p{White_Space}])|\p{White_Space}+/gu;
const specials = ["<|endoftext|>", "<|user|>", "<|assistant|>", "<|end|>"];
export function tokenizer(text) {
const pairs = text.trim().split(/\r?\n/).map(line => line.split(" ").map(Number));
const ranks = new Map();
const vocab = Array.from({ length: 256 }, (_, i) => new Uint8Array([i]));
for (const [a, b] of pairs) {
ranks.set(`${a},${b}`, vocab.length);
vocab.push(new Uint8Array([...vocab[a], ...vocab[b]]));
}
const special = Object.fromEntries(specials.map((s, i) => [s, vocab.length + i]));
const encoder = new TextEncoder();
const split = /(<\|endoftext\|>|<\|user\|>|<\|assistant\|>|<\|end\|>)/g;
function encode(text) {
const result = [];
for (const part of text.split(split)) {
if (part in special) {
result.push(special[part]);
continue;
}
for (const match of part.matchAll(pattern)) {
let ids = Array.from(encoder.encode(match[0]));
while (ids.length > 1) {
let rank = Infinity;
let a, b;
for (let i = 0; i < ids.length - 1; i++) {
const value = ranks.get(`${ids[i]},${ids[i + 1]}`) ?? Infinity;
if (value < rank) { rank = value; a = ids[i]; b = ids[i + 1]; }
}
if (!Number.isFinite(rank)) break;
const merged = [];
for (let i = 0; i < ids.length; i++) {
if (ids[i] === a && ids[i + 1] === b) { merged.push(rank); i++; }
else merged.push(ids[i]);
}
ids = merged;
}
result.push(...ids);
}
}
return result;
}
return { encode, vocab, special };
}
export function storyPrompt(fields) {
const names = ["Summary", "Words", "Random sentence", "Features"];
if (Object.keys(fields).some(key => !names.includes(key)) ||
Object.values(fields).some(value => typeof value !== "string")) {
throw new Error("Use the supported story fields with text values.");
}
if (Object.values(fields).reduce((n, value) => n + value.length, 0) > 4096) {
throw new Error("Your instructions are too long. Try a shorter story idea.");
}
if (Object.values(fields).some(value => specials.some(s => value.includes(s)))) {
throw new Error("Remove reserved tokens such as <|user|> from your instructions.");
}
const clean = names.map(key => [key, (fields[key] ?? "").replace(/[\p{White_Space}\u001c-\u001f]+/gu, " ").replace(/^ | $/g, "")]);
const lines = clean.filter(([, value]) => value).map(([key, value]) => `${key}: ${value}`);
if (!lines.length) throw new Error("Enter a story idea or a word to include.");
return `<|user|>\nWrite a short story.\n${lines.join("\n")}<|end|>\n<|assistant|>\n`;
}
|