Spaces:
Running on Zero
Running on Zero
atakan Claude Sonnet 5 commited on
Commit ·
ad33173
1
Parent(s): 6be46a5
fix: Increase llama.cpp context window to fit the system prompt + tool schemas
Browse filesThe system prompt plus all 29 registered tool JSON schemas alone total
~6,436 tokens, which already exceeded the previous n_ctx=4096 before a
single user message or generated token was added. Every request was
failing immediately with "Requested tokens exceed context window".
Bumped to 16384 to leave headroom for multi-step tool-calling and
generation.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
controlai_agent/orchestrator.py
CHANGED
|
@@ -171,7 +171,7 @@ class ControlAIAgent:
|
|
| 171 |
self.llama_model = llama_cpp.Llama(
|
| 172 |
model_path=str(model_path),
|
| 173 |
n_gpu_layers=-1, # Offload all layers to Metal / CUDA GPU
|
| 174 |
-
n_ctx=
|
| 175 |
verbose=False,
|
| 176 |
)
|
| 177 |
self.hf_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-3B-Instruct", trust_remote_code=True)
|
|
@@ -196,7 +196,7 @@ class ControlAIAgent:
|
|
| 196 |
self.llama_model = llama_cpp.Llama.from_pretrained(
|
| 197 |
repo_id=gguf_repo,
|
| 198 |
filename=gguf_filename,
|
| 199 |
-
n_ctx=
|
| 200 |
n_threads=threads,
|
| 201 |
verbose=False,
|
| 202 |
)
|
|
|
|
| 171 |
self.llama_model = llama_cpp.Llama(
|
| 172 |
model_path=str(model_path),
|
| 173 |
n_gpu_layers=-1, # Offload all layers to Metal / CUDA GPU
|
| 174 |
+
n_ctx=16384,
|
| 175 |
verbose=False,
|
| 176 |
)
|
| 177 |
self.hf_tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen2.5-3B-Instruct", trust_remote_code=True)
|
|
|
|
| 196 |
self.llama_model = llama_cpp.Llama.from_pretrained(
|
| 197 |
repo_id=gguf_repo,
|
| 198 |
filename=gguf_filename,
|
| 199 |
+
n_ctx=16384,
|
| 200 |
n_threads=threads,
|
| 201 |
verbose=False,
|
| 202 |
)
|