test_cpu.py

import time
import torch
from transformers import AutoModelForCausalLM, AutoTokenizer
MODEL_ID = "HuggingFaceTB/SmolLM2-360M"
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.float32, low_cpu_mem_usage=True)
prompt = "λόγος\n"
inputs = tokenizer(prompt, return_tensors="pt")
start = time.time()
with torch.no_grad():
    model.generate(**inputs, max_new_tokens=100, do_sample=True, temperature=1.2, top_p=0.95)
print(f"Time taken: {time.time() - start:.2f}s")