Mehdi commited on
Commit
8f2e039
·
1 Parent(s): c8e8b73

fix: enable sampling — temperature=0.8 for MCQ, 0.4 for evaluator

Browse files

Greedy decoding (do_sample=False) caused the model to repeat the same
answer for all 4 MCQ options (e.g. 4x BERT) and to copy source text
verbatim for the model answer. Sampling breaks the repetition loop and
produces diverse, coherent outputs.

Files changed (3) hide show
  1. core/evaluator.py +1 -1
  2. core/questioner.py +1 -1
  3. model/llm.py +7 -4
core/evaluator.py CHANGED
@@ -50,4 +50,4 @@ def evaluate_answer(question: str, chunk: str, student_answer: str, language: st
50
  )
51
  # 4-section feedback fits comfortably in 320 tokens — keeps CPU
52
  # (llama.cpp) latency inside the UI timeout.
53
- return llm.generate(prompt, max_new_tokens=320).strip()
 
50
  )
51
  # 4-section feedback fits comfortably in 320 tokens — keeps CPU
52
  # (llama.cpp) latency inside the UI timeout.
53
+ return llm.generate(prompt, max_new_tokens=320, temperature=0.4).strip()
core/questioner.py CHANGED
@@ -96,7 +96,7 @@ def generate_mcq(chunk: str, language: str = "English") -> dict:
96
  prompt = _MCQ_TEMPLATE.format(chunk=chunk.strip(), language=language)
97
  mcq: dict = {}
98
  for _ in range(3):
99
- raw = llm.generate(prompt).strip()
100
  mcq = parse_mcq(raw)
101
  choices = list(mcq.get("choices", {}).values())
102
  if len(choices) == 4 and len({c.lower().strip() for c in choices}) == 4:
 
96
  prompt = _MCQ_TEMPLATE.format(chunk=chunk.strip(), language=language)
97
  mcq: dict = {}
98
  for _ in range(3):
99
+ raw = llm.generate(prompt, temperature=0.8).strip()
100
  mcq = parse_mcq(raw)
101
  choices = list(mcq.get("choices", {}).values())
102
  if len(choices) == 4 and len({c.lower().strip() for c in choices}) == 4:
model/llm.py CHANGED
@@ -90,17 +90,20 @@ class LLM:
90
  tokenizer=self._tokenizer,
91
  )
92
 
93
- def generate(self, prompt: str, max_new_tokens: int = DEFAULT_MAX_NEW_TOKENS) -> str:
94
  """Run *prompt* through the model and return the generated text only."""
95
  messages = [{"role": "user", "content": prompt}]
96
  text = self._tokenizer.apply_chat_template(
97
  messages, tokenize=False, add_generation_prompt=True,
98
  enable_thinking=False,
99
  )
 
100
  output = self._pipe(
101
  text,
102
  max_new_tokens=max_new_tokens,
103
- do_sample=False,
 
 
104
  return_full_text=False,
105
  )
106
  return output[0]["generated_text"]
@@ -131,11 +134,11 @@ class LlamaCppLLM:
131
  verbose=False,
132
  )
133
 
134
- def generate(self, prompt: str, max_new_tokens: int = DEFAULT_MAX_NEW_TOKENS) -> str:
135
  out = self._llm.create_chat_completion(
136
  messages=[{"role": "user", "content": prompt}],
137
  max_tokens=max_new_tokens,
138
- temperature=0.0,
139
  )
140
  return out["choices"][0]["message"]["content"]
141
 
 
90
  tokenizer=self._tokenizer,
91
  )
92
 
93
+ def generate(self, prompt: str, max_new_tokens: int = DEFAULT_MAX_NEW_TOKENS, temperature: float = 0.0) -> str:
94
  """Run *prompt* through the model and return the generated text only."""
95
  messages = [{"role": "user", "content": prompt}]
96
  text = self._tokenizer.apply_chat_template(
97
  messages, tokenize=False, add_generation_prompt=True,
98
  enable_thinking=False,
99
  )
100
+ sample = temperature > 0.0
101
  output = self._pipe(
102
  text,
103
  max_new_tokens=max_new_tokens,
104
+ do_sample=sample,
105
+ temperature=temperature if sample else None,
106
+ top_p=0.95 if sample else None,
107
  return_full_text=False,
108
  )
109
  return output[0]["generated_text"]
 
134
  verbose=False,
135
  )
136
 
137
+ def generate(self, prompt: str, max_new_tokens: int = DEFAULT_MAX_NEW_TOKENS, temperature: float = 0.0) -> str:
138
  out = self._llm.create_chat_completion(
139
  messages=[{"role": "user", "content": prompt}],
140
  max_tokens=max_new_tokens,
141
+ temperature=temperature,
142
  )
143
  return out["choices"][0]["message"]["content"]
144