Instructions to use tmickleydoyle/tiny-autocomplete with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use tmickleydoyle/tiny-autocomplete with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="tmickleydoyle/tiny-autocomplete")

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("tmickleydoyle/tiny-autocomplete")
model = AutoModelForCausalLM.from_pretrained("tmickleydoyle/tiny-autocomplete", device_map="auto")

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use tmickleydoyle/tiny-autocomplete with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "tmickleydoyle/tiny-autocomplete"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tmickleydoyle/tiny-autocomplete",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Use Docker

docker model run hf.co/tmickleydoyle/tiny-autocomplete

SGLang

How to use tmickleydoyle/tiny-autocomplete with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "tmickleydoyle/tiny-autocomplete" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tmickleydoyle/tiny-autocomplete",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "tmickleydoyle/tiny-autocomplete" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tmickleydoyle/tiny-autocomplete",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Docker Model Runner
How to use tmickleydoyle/tiny-autocomplete with Docker Model Runner:
```
docker model run hf.co/tmickleydoyle/tiny-autocomplete
```

tiny-autocomplete / web /demo.ts

tmickleydoyle

Confidence-gated ghost text for coding prompts: Kneser-Ney compile, 4.5 MB browser export + zero-dep runtime, held-out precision/keystrokes-saved benchmarks

4113449 verified about 12 hours ago

Raw History Blame Contribute Delete

3.32 kB

// Static demo for web/autocomplete.ts. Bundle with:
//   bun build web/demo.ts --outfile web/demo.js --target browser --minify

import { CompiledAutocomplete, type Policy } from "./autocomplete.ts";

const $ = <T extends HTMLElement>(id: string) => document.getElementById(id) as T;
const promptEl = $<HTMLTextAreaElement>("prompt");
const ghostEl = $("ghost");
const statusEl = $("status");
const guessEl = $("guess");
const tauEl = $<HTMLInputElement>("tau");
const tauLabel = $("tau-label");
const loadEl = $("load");

const EXAMPLES = [
  "write a python fun", "how do i ", "Write a function to find the ", "how to convert a ",
  "can you explain ", "I hope this email ", "Once upon a ", "what is the difference between ",
];

function esc(s: string): string {
  return s.replace(/[&<>]/g, (c) => ({ "&": "&amp;", "<": "&lt;", ">": "&gt;" })[c]!);
}

async function main(): Promise<void> {
  const t0 = performance.now();
  const ac = await CompiledAutocomplete.load("model.bin", (f) => (loadEl.textContent = `downloading model... ${Math.round(f * 100)}%`));
  const loadMs = performance.now() - t0;
  const policy: Policy = { ...ac.policy };
  tauEl.value = String(policy.tau);
  tauLabel.textContent = `${Math.round(policy.tau * 100)}%`;
  loadEl.textContent =
    `${(ac.bytes / 1e6).toFixed(1)} MB · ${ac.contexts.toLocaleString()} compiled contexts · ` +
    `${ac.order}-token context · ${ac.vocab.length.toLocaleString()} words · loaded in ${Math.round(loadMs)} ms`;
  promptEl.disabled = false;
  promptEl.focus();

  let ghost = "";
  const render = () => {
    const text = promptEl.value;
    const start = performance.now();
    const s = ac.suggest(text, policy);
    const ms = performance.now() - start;
    ghost = s.ghost;
    ghostEl.innerHTML = esc(text) + "<span>" + esc(ghost) + "</span>";
    promptEl.style.height = "auto";
    promptEl.style.height = Math.max(promptEl.scrollHeight, ghostEl.scrollHeight) + "px";
    statusEl.textContent = text ? `${ms.toFixed(2)} ms` : "";
    // First step that adds text (the first step is empty when the typed word is already complete).
    const i = s.steps.findIndex((st) => st.text);
    const guess = i < 0 ? "" : s.steps.slice(0, i + 1).map((st) => st.text).join("");
    if (!text) guessEl.textContent = "";
    else if (ghost) guessEl.textContent = `suggesting with ${Math.round(s.prob * 100)}% confidence`;
    else if (guess) guessEl.textContent = `staying quiet: best guess "${guess}" is only ${Math.round(s.steps[i].prob * 100)}% likely`;
    else guessEl.textContent = "staying quiet: nothing known for this prefix";
  };

  promptEl.addEventListener("input", render);
  promptEl.addEventListener("keydown", (e) => {
    if (e.key === "Tab" && ghost) {
      e.preventDefault();
      promptEl.value += ghost;
      render();
    }
  });
  tauEl.addEventListener("input", () => {
    policy.tau = Number(tauEl.value);
    tauLabel.textContent = `${Math.round(policy.tau * 100)}%`;
    render();
  });
  for (const ex of EXAMPLES) {
    const b = document.createElement("button");
    b.className = "ex";
    b.textContent = ex.trim();
    b.onclick = () => {
      promptEl.value = ex;
      render();
      promptEl.focus();
    };
    $("examples").appendChild(b);
  }
  render();
}

main().catch((e) => (loadEl.textContent = String(e)));

	// Static demo for web/autocomplete.ts. Bundle with:
	// bun build web/demo.ts --outfile web/demo.js --target browser --minify

	import { CompiledAutocomplete, type Policy } from "./autocomplete.ts";

	const $ = <T extends HTMLElement>(id: string) => document.getElementById(id) as T;
	const promptEl = $<HTMLTextAreaElement>("prompt");
	const ghostEl = $("ghost");
	const statusEl = $("status");
	const guessEl = $("guess");
	const tauEl = $<HTMLInputElement>("tau");
	const tauLabel = $("tau-label");
	const loadEl = $("load");

	const EXAMPLES = [
	"write a python fun", "how do i ", "Write a function to find the ", "how to convert a ",
	"can you explain ", "I hope this email ", "Once upon a ", "what is the difference between ",
	];

	function esc(s: string): string {
	return s.replace(/[&<>]/g, (c) => ({ "&": "&", "<": "<", ">": ">" })[c]!);
	}

	async function main(): Promise<void> {
	const t0 = performance.now();
	const ac = await CompiledAutocomplete.load("model.bin", (f) => (loadEl.textContent = `downloading model... ${Math.round(f * 100)}%`));
	const loadMs = performance.now() - t0;
	const policy: Policy = { ...ac.policy };
	tauEl.value = String(policy.tau);
	tauLabel.textContent = `${Math.round(policy.tau * 100)}%`;
	loadEl.textContent =
	`${(ac.bytes / 1e6).toFixed(1)} MB · ${ac.contexts.toLocaleString()} compiled contexts · ` +
	`${ac.order}-token context · ${ac.vocab.length.toLocaleString()} words · loaded in ${Math.round(loadMs)} ms`;
	promptEl.disabled = false;
	promptEl.focus();

	let ghost = "";
	const render = () => {
	const text = promptEl.value;
	const start = performance.now();
	const s = ac.suggest(text, policy);
	const ms = performance.now() - start;
	ghost = s.ghost;
	ghostEl.innerHTML = esc(text) + "<span>" + esc(ghost) + "</span>";
	promptEl.style.height = "auto";
	promptEl.style.height = Math.max(promptEl.scrollHeight, ghostEl.scrollHeight) + "px";
	statusEl.textContent = text ? `${ms.toFixed(2)} ms` : "";
	// First step that adds text (the first step is empty when the typed word is already complete).
	const i = s.steps.findIndex((st) => st.text);
	const guess = i < 0 ? "" : s.steps.slice(0, i + 1).map((st) => st.text).join("");
	if (!text) guessEl.textContent = "";
	else if (ghost) guessEl.textContent = `suggesting with ${Math.round(s.prob * 100)}% confidence`;
	else if (guess) guessEl.textContent = `staying quiet: best guess "${guess}" is only ${Math.round(s.steps[i].prob * 100)}% likely`;
	else guessEl.textContent = "staying quiet: nothing known for this prefix";
	};

	promptEl.addEventListener("input", render);
	promptEl.addEventListener("keydown", (e) => {
	if (e.key === "Tab" && ghost) {
	e.preventDefault();
	promptEl.value += ghost;
	render();
	}
	});
	tauEl.addEventListener("input", () => {
	policy.tau = Number(tauEl.value);
	tauLabel.textContent = `${Math.round(policy.tau * 100)}%`;
	render();
	});
	for (const ex of EXAMPLES) {
	const b = document.createElement("button");
	b.className = "ex";
	b.textContent = ex.trim();
	b.onclick = () => {
	promptEl.value = ex;
	render();
	promptEl.focus();
	};
	$("examples").appendChild(b);
	}
	render();
	}

	main().catch((e) => (loadEl.textContent = String(e)));