Instructions to use tmickleydoyle/tiny-autocomplete with libraries, inference providers, notebooks, and local apps. Follow these links to get started.

Libraries

How to use tmickleydoyle/tiny-autocomplete with Transformers:

# Use a pipeline as a high-level helper
from transformers import pipeline

pipe = pipeline("text-generation", model="tmickleydoyle/tiny-autocomplete")

# pip install -U transformers accelerate
# Load model directly
from transformers import AutoTokenizer, AutoModelForCausalLM

tokenizer = AutoTokenizer.from_pretrained("tmickleydoyle/tiny-autocomplete")
model = AutoModelForCausalLM.from_pretrained("tmickleydoyle/tiny-autocomplete", device_map="auto")

Notebooks
Google Colab
Kaggle
Local Apps Settings

vLLM

How to use tmickleydoyle/tiny-autocomplete with vLLM:

Install from pip and serve model

# Install vLLM from pip:
pip install vllm
# Start the vLLM server:
vllm serve "tmickleydoyle/tiny-autocomplete"
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:8000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tmickleydoyle/tiny-autocomplete",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Use Docker

docker model run hf.co/tmickleydoyle/tiny-autocomplete

SGLang

How to use tmickleydoyle/tiny-autocomplete with SGLang:

Install from pip and serve model

# Install SGLang from pip:
pip install sglang
# Start the SGLang server:
python3 -m sglang.launch_server \
    --model-path "tmickleydoyle/tiny-autocomplete" \
    --host 0.0.0.0 \
    --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tmickleydoyle/tiny-autocomplete",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Use Docker images

docker run --gpus all \
    --shm-size 32g \
    -p 30000:30000 \
    -v ~/.cache/huggingface:/root/.cache/huggingface \
    --env "HF_TOKEN=<secret>" \
    --ipc=host \
    lmsysorg/sglang:latest \
    python3 -m sglang.launch_server \
        --model-path "tmickleydoyle/tiny-autocomplete" \
        --host 0.0.0.0 \
        --port 30000
# Call the server using curl (OpenAI-compatible API):
curl -X POST "http://localhost:30000/v1/completions" \
	-H "Content-Type: application/json" \
	--data '{
		"model": "tmickleydoyle/tiny-autocomplete",
		"prompt": "Once upon a time,",
		"max_tokens": 512,
		"temperature": 0.5
	}'

Docker Model Runner
How to use tmickleydoyle/tiny-autocomplete with Docker Model Runner:
```
docker model run hf.co/tmickleydoyle/tiny-autocomplete
```

tiny-autocomplete / ui.ts

tmickleydoyle

Compiled (not trained) 91M-param Llama autocomplete model, TypeScript runtime and UI, docs

ab4f92b verified 1 day ago

Raw History Blame Contribute Delete

2.16 kB

// Browser UI for trying prompts (zero dependencies).
//
//   bun ui.ts          then open http://localhost:3000
//   PORT=8080 node ui.ts
//
// Serves ui.html and a JSON endpoint backed by the plain-TypeScript model in model.ts.

import { readFileSync } from "node:fs";
import { createServer } from "node:http";
import { join } from "node:path";
import { Autocompleter } from "./model.ts";

const DIR = import.meta.dirname ?? new URL(".", import.meta.url).pathname;
const PORT = Number(process.env.PORT ?? 3000);

console.log("Loading model...");
const ac = new Autocompleter(DIR);
const html = readFileSync(join(DIR, "ui.html"));
const cfg = ac.model.cfg;
const info = {
  params: ac.model.numParams,
  vocab: cfg.vocab_size,
  layers: cfg.num_hidden_layers,
  hidden: cfg.hidden_size,
  neurons: cfg.intermediate_size,
};

function json(res: import("node:http").ServerResponse, status: number, body: unknown): void {
  res.writeHead(status, { "content-type": "application/json" });
  res.end(JSON.stringify(body));
}

createServer((req, res) => {
  if (req.method === "GET" && (req.url === "/" || req.url === "/index.html")) {
    res.writeHead(200, { "content-type": "text/html; charset=utf-8" });
    res.end(html);
  } else if (req.method === "GET" && req.url === "/api/info") {
    json(res, 200, info);
  } else if (req.method === "POST" && req.url === "/api/complete") {
    let body = "";
    req.on("data", (chunk) => (body += chunk));
    req.on("end", () => {
      try {
        const { prompt, topK } = JSON.parse(body || "{}");
        if (typeof prompt !== "string" || prompt.length > 1000) return json(res, 400, { error: "prompt must be a string under 1000 chars" });
        const t0 = performance.now();
        const result = prompt.trim() ? ac.complete(prompt, Math.min(Number(topK) || 5, 20)) : { prompt, completion: "", top: [] };
        json(res, 200, { ...result, ms: Math.round(performance.now() - t0) });
      } catch (e) {
        json(res, 400, { error: String(e) });
      }
    });
  } else {
    res.writeHead(404).end();
  }
}).listen(PORT, "127.0.0.1", () => console.log(`tiny-autocomplete UI: http://localhost:${PORT}`));

	// Browser UI for trying prompts (zero dependencies).
	//
	// bun ui.ts then open http://localhost:3000
	// PORT=8080 node ui.ts
	//
	// Serves ui.html and a JSON endpoint backed by the plain-TypeScript model in model.ts.

	import { readFileSync } from "node:fs";
	import { createServer } from "node:http";
	import { join } from "node:path";
	import { Autocompleter } from "./model.ts";

	const DIR = import.meta.dirname ?? new URL(".", import.meta.url).pathname;
	const PORT = Number(process.env.PORT ?? 3000);

	console.log("Loading model...");
	const ac = new Autocompleter(DIR);
	const html = readFileSync(join(DIR, "ui.html"));
	const cfg = ac.model.cfg;
	const info = {
	params: ac.model.numParams,
	vocab: cfg.vocab_size,
	layers: cfg.num_hidden_layers,
	hidden: cfg.hidden_size,
	neurons: cfg.intermediate_size,
	};

	function json(res: import("node:http").ServerResponse, status: number, body: unknown): void {
	res.writeHead(status, { "content-type": "application/json" });
	res.end(JSON.stringify(body));
	}

	createServer((req, res) => {
	if (req.method === "GET" && (req.url === "/" \|\| req.url === "/index.html")) {
	res.writeHead(200, { "content-type": "text/html; charset=utf-8" });
	res.end(html);
	} else if (req.method === "GET" && req.url === "/api/info") {
	json(res, 200, info);
	} else if (req.method === "POST" && req.url === "/api/complete") {
	let body = "";
	req.on("data", (chunk) => (body += chunk));
	req.on("end", () => {
	try {
	const { prompt, topK } = JSON.parse(body \|\| "{}");
	if (typeof prompt !== "string" \|\| prompt.length > 1000) return json(res, 400, { error: "prompt must be a string under 1000 chars" });
	const t0 = performance.now();
	const result = prompt.trim() ? ac.complete(prompt, Math.min(Number(topK) \|\| 5, 20)) : { prompt, completion: "", top: [] };
	json(res, 200, { ...result, ms: Math.round(performance.now() - t0) });
	} catch (e) {
	json(res, 400, { error: String(e) });
	}
	});
	} else {
	res.writeHead(404).end();
	}
	}).listen(PORT, "127.0.0.1", () => console.log(`tiny-autocomplete UI: http://localhost:${PORT}`));