Spaces:
Running on Zero
Running on Zero
atakan Claude Sonnet 5 commited on
Commit ·
dfe7123
1
Parent(s): 9b37428
add: CONTROLAI_BACKEND=pytorch escape hatch to bypass the GGUF path
Browse filesNeeded to measure the PyTorch/CUDA path against GGUF on real Space hardware:
the deployed GGUF Q8_0 path (fast and correct in local testing, 8-19s/turn)
took over 7.5 minutes for the same prompt on the live Space and was aborted
by ZeroGPU's 300s budget -- so before trusting either backend as the
default, measure both on the actual target hardware rather than assume.
Also fixed two leftover "Q4_K_M" log strings that no longer matched the
actual Q8_0 default.
Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>
controlai_agent/orchestrator.py
CHANGED
|
@@ -616,12 +616,16 @@ class ControlAIAgent:
|
|
| 616 |
# f16 to be considered near-lossless, at roughly half f16's size.
|
| 617 |
gguf_repo = os.environ.get("CONTROLAI_GGUF_REPO", CONTROLAI_HF_REPO)
|
| 618 |
gguf_filename = os.environ.get("CONTROLAI_GGUF_FILENAME", "*controlai-q8_0.gguf")
|
|
|
|
|
|
|
|
|
|
|
|
|
| 619 |
|
| 620 |
self.llama_model = None
|
| 621 |
-
if HAS_LLAMA_CPP:
|
| 622 |
try:
|
| 623 |
threads = min(os.cpu_count() or 2, 4)
|
| 624 |
-
print(f"Loading
|
| 625 |
self.llama_model = llama_cpp.Llama.from_pretrained(
|
| 626 |
repo_id=gguf_repo,
|
| 627 |
filename=gguf_filename,
|
|
@@ -631,7 +635,7 @@ class ControlAIAgent:
|
|
| 631 |
)
|
| 632 |
self.is_gguf = True
|
| 633 |
self.hf_tokenizer = AutoTokenizer.from_pretrained(CONTROLAI_HF_REPO, trust_remote_code=True)
|
| 634 |
-
print("GGUF
|
| 635 |
except Exception as exc:
|
| 636 |
print(f"Notice: llama_cpp GGUF auto-load failed, falling back to PyTorch: {exc}")
|
| 637 |
self.is_gguf = False
|
|
|
|
| 616 |
# f16 to be considered near-lossless, at roughly half f16's size.
|
| 617 |
gguf_repo = os.environ.get("CONTROLAI_GGUF_REPO", CONTROLAI_HF_REPO)
|
| 618 |
gguf_filename = os.environ.get("CONTROLAI_GGUF_FILENAME", "*controlai-q8_0.gguf")
|
| 619 |
+
# Escape hatch for measuring the PyTorch/CUDA path against the GGUF
|
| 620 |
+
# path on real deployment hardware -- set to "pytorch" to skip the
|
| 621 |
+
# GGUF attempt entirely and go straight to the branch below.
|
| 622 |
+
force_backend = os.environ.get("CONTROLAI_BACKEND", "").lower()
|
| 623 |
|
| 624 |
self.llama_model = None
|
| 625 |
+
if HAS_LLAMA_CPP and force_backend != "pytorch":
|
| 626 |
try:
|
| 627 |
threads = min(os.cpu_count() or 2, 4)
|
| 628 |
+
print(f"Loading GGUF ControlAI model ({gguf_filename}) from {gguf_repo} (threads: {threads})...")
|
| 629 |
self.llama_model = llama_cpp.Llama.from_pretrained(
|
| 630 |
repo_id=gguf_repo,
|
| 631 |
filename=gguf_filename,
|
|
|
|
| 635 |
)
|
| 636 |
self.is_gguf = True
|
| 637 |
self.hf_tokenizer = AutoTokenizer.from_pretrained(CONTROLAI_HF_REPO, trust_remote_code=True)
|
| 638 |
+
print("GGUF ControlAI model loaded successfully via llama_cpp.")
|
| 639 |
except Exception as exc:
|
| 640 |
print(f"Notice: llama_cpp GGUF auto-load failed, falling back to PyTorch: {exc}")
|
| 641 |
self.is_gguf = False
|