atakan Claude Sonnet 5 commited on
Commit
dfe7123
·
1 Parent(s): 9b37428

add: CONTROLAI_BACKEND=pytorch escape hatch to bypass the GGUF path

Browse files

Needed to measure the PyTorch/CUDA path against GGUF on real Space hardware:
the deployed GGUF Q8_0 path (fast and correct in local testing, 8-19s/turn)
took over 7.5 minutes for the same prompt on the live Space and was aborted
by ZeroGPU's 300s budget -- so before trusting either backend as the
default, measure both on the actual target hardware rather than assume.

Also fixed two leftover "Q4_K_M" log strings that no longer matched the
actual Q8_0 default.

Co-Authored-By: Claude Sonnet 5 <noreply@anthropic.com>

Files changed (1) hide show
  1. controlai_agent/orchestrator.py +7 -3
controlai_agent/orchestrator.py CHANGED
@@ -616,12 +616,16 @@ class ControlAIAgent:
616
  # f16 to be considered near-lossless, at roughly half f16's size.
617
  gguf_repo = os.environ.get("CONTROLAI_GGUF_REPO", CONTROLAI_HF_REPO)
618
  gguf_filename = os.environ.get("CONTROLAI_GGUF_FILENAME", "*controlai-q8_0.gguf")
 
 
 
 
619
 
620
  self.llama_model = None
621
- if HAS_LLAMA_CPP:
622
  try:
623
  threads = min(os.cpu_count() or 2, 4)
624
- print(f"Loading high-speed GGUF Q4_K_M ControlAI model from {gguf_repo} (threads: {threads})...")
625
  self.llama_model = llama_cpp.Llama.from_pretrained(
626
  repo_id=gguf_repo,
627
  filename=gguf_filename,
@@ -631,7 +635,7 @@ class ControlAIAgent:
631
  )
632
  self.is_gguf = True
633
  self.hf_tokenizer = AutoTokenizer.from_pretrained(CONTROLAI_HF_REPO, trust_remote_code=True)
634
- print("GGUF Q4_K_M ControlAI model loaded successfully via llama_cpp.")
635
  except Exception as exc:
636
  print(f"Notice: llama_cpp GGUF auto-load failed, falling back to PyTorch: {exc}")
637
  self.is_gguf = False
 
616
  # f16 to be considered near-lossless, at roughly half f16's size.
617
  gguf_repo = os.environ.get("CONTROLAI_GGUF_REPO", CONTROLAI_HF_REPO)
618
  gguf_filename = os.environ.get("CONTROLAI_GGUF_FILENAME", "*controlai-q8_0.gguf")
619
+ # Escape hatch for measuring the PyTorch/CUDA path against the GGUF
620
+ # path on real deployment hardware -- set to "pytorch" to skip the
621
+ # GGUF attempt entirely and go straight to the branch below.
622
+ force_backend = os.environ.get("CONTROLAI_BACKEND", "").lower()
623
 
624
  self.llama_model = None
625
+ if HAS_LLAMA_CPP and force_backend != "pytorch":
626
  try:
627
  threads = min(os.cpu_count() or 2, 4)
628
+ print(f"Loading GGUF ControlAI model ({gguf_filename}) from {gguf_repo} (threads: {threads})...")
629
  self.llama_model = llama_cpp.Llama.from_pretrained(
630
  repo_id=gguf_repo,
631
  filename=gguf_filename,
 
635
  )
636
  self.is_gguf = True
637
  self.hf_tokenizer = AutoTokenizer.from_pretrained(CONTROLAI_HF_REPO, trust_remote_code=True)
638
+ print("GGUF ControlAI model loaded successfully via llama_cpp.")
639
  except Exception as exc:
640
  print(f"Notice: llama_cpp GGUF auto-load failed, falling back to PyTorch: {exc}")
641
  self.is_gguf = False