Serve with greedy decoding: sampling hallucinates on this lightly-tuned checkpoint

app.py

@@ -115,10 +115,13 @@ def _prep(req: "GenReq"):
     if ids is None or not hasattr(ids, "shape"):
         ids = tok(prompt, return_tensors="pt").input_ids
     ids = ids.to(dev)  # batch_size = 1 ONLY
-    temp = float(req.temperature if req.temperature is not None else 0.7)
+    # Greedy decoding is enforced. This checkpoint carries only light instruction
+    # tuning, so sampling drifts off-distribution and hallucinates (e.g. inventing a
+    # company profile when asked who it is). Deterministic decoding reproduces what
+    # was actually trained. `temperature` is accepted for API compatibility but ignored.
     gen = dict(
         max_new_tokens=min(int(req.max_new_tokens or MAX_NEW), MAX_NEW),
-        do_sample=temp > 0, temperature=max(temp, 0.01), top_p=0.9,
+        do_sample=False,
         repetition_penalty=1.3, no_repeat_ngram_size=3, use_cache=False,
         pad_token_id=getattr(tok, "eos_token_id", None),
     )