Code-Father commited on
Commit
060c03b
·
1 Parent(s): 34cfc81

updated infrance to remove thinking token

Browse files
Files changed (1) hide show
  1. inference.py +10 -1
inference.py CHANGED
@@ -29,6 +29,8 @@ from models import (
29
  StoreObservation,
30
  dump_model,
31
  )
 
 
32
 
33
  API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
34
  MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen3.5-397B-A17B")
@@ -269,11 +271,18 @@ async def get_model_action(
269
  {"role": "user", "content": user_msg},
270
  ],
271
  temperature=0.3,
272
- max_tokens=1024,
273
  )
274
 
275
  content = response.choices[0].message.content or "{}"
276
  content = content.strip()
 
 
 
 
 
 
 
277
  if content.startswith("```"):
278
  content = content.split("\n", 1)[-1]
279
  if content.endswith("```"):
 
29
  StoreObservation,
30
  dump_model,
31
  )
32
+ from dotenv import load_dotenv
33
+ load_dotenv()
34
 
35
  API_BASE_URL = os.getenv("API_BASE_URL", "https://router.huggingface.co/v1")
36
  MODEL_NAME = os.getenv("MODEL_NAME", "Qwen/Qwen3.5-397B-A17B")
 
271
  {"role": "user", "content": user_msg},
272
  ],
273
  temperature=0.3,
274
+ max_tokens=4096,
275
  )
276
 
277
  content = response.choices[0].message.content or "{}"
278
  content = content.strip()
279
+
280
+ print(f"[DEBUG] Raw LLM response (first 500 chars): {content[:500]}", flush=True)
281
+
282
+ think_end = content.find("</think>")
283
+ if think_end != -1:
284
+ content = content[think_end + len("</think>"):].strip()
285
+
286
  if content.startswith("```"):
287
  content = content.split("\n", 1)[-1]
288
  if content.endswith("```"):