Lucas
Fix Space prefill slowness: cap prefill threads, warm up at startup
689f982
Raw
History Blame Contribute Delete
2.36 kB
import logging
import os
from training_coach.models import ParsedCheckIn
from training_coach.parser_llama_cpp import (
parse_check_in_with_llama_cpp,
warm_up_llama_cpp_parser,
)
from training_coach.parser_ollama import parse_check_in_with_ollama
from training_coach.parser_runtime import parse_check_in_with_model
DEFAULT_LOCAL_BACKEND = "ollama"
DEFAULT_SPACE_BACKEND = "llama_cpp"
DEFAULT_OLLAMA_MODEL = "qwen3:1.7B"
logger = logging.getLogger(__name__)
def parser_backend() -> str:
configured_backend = os.getenv("PARSER_BACKEND", "").strip().lower()
if configured_backend:
return configured_backend
if os.getenv("SPACE_ID"):
return DEFAULT_SPACE_BACKEND
return DEFAULT_LOCAL_BACKEND
def warm_up_parser_backend() -> None:
if parser_backend() == "llama_cpp":
warm_up_llama_cpp_parser()
def parse_check_in_with_configured_backend(raw_text: str) -> ParsedCheckIn:
backend = parser_backend()
logger.info(
"event=parser_start backend=%s text_chars=%s",
backend,
len(raw_text),
)
if backend == "ollama":
parsed = parse_check_in_with_ollama(
raw_text,
model_name=os.getenv("OLLAMA_MODEL", DEFAULT_OLLAMA_MODEL),
)
logger.info(
"event=parser_complete backend=%s missing_fields=%s follow_up_questions=%s",
backend,
len(parsed.missing_fields),
len(parsed.follow_up_questions),
)
return parsed
if backend == "llama_cpp":
parsed = parse_check_in_with_llama_cpp(raw_text)
logger.info(
"event=parser_complete backend=%s missing_fields=%s follow_up_questions=%s",
backend,
len(parsed.missing_fields),
len(parsed.follow_up_questions),
)
return parsed
if backend == "transformers":
parsed = parse_check_in_with_model(raw_text)
logger.info(
"event=parser_complete backend=%s missing_fields=%s follow_up_questions=%s",
backend,
len(parsed.missing_fields),
len(parsed.follow_up_questions),
)
return parsed
logger.error("event=parser_unsupported_backend backend=%s", backend)
raise ValueError(
"Unsupported PARSER_BACKEND. Use 'ollama', 'llama_cpp', or 'transformers'."
)