Spaces:
Sleeping
Sleeping
File size: 1,672 Bytes
c31f4d7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 | """Compare candidate models on speed and valid JSON. Run before choosing MODEL_ID.
Usage: python scripts/probe_models.py [model ...]
"""
from __future__ import annotations
import os
import sys
import time
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from dotenv import load_dotenv
from pydantic import BaseModel
from core import llm
CANDIDATES = [
"meta-llama/Llama-3.3-70B-Instruct",
"Qwen/Qwen3-Next-80B-A3B-Instruct",
"Qwen/Qwen3-235B-A22B-Instruct-2507",
"openai/gpt-oss-120b",
"deepseek-ai/DeepSeek-V3.2",
]
PROMPT = [
{"role": "system", "content": "Reply with only a JSON object with keys: message (string, under 40 words), stage (one of opening, exploring, negotiating, verdict), wants_to_end (boolean)."},
{"role": "user", "content": "You are a founder's agent opening a conversation with an investor's agent about a Seed round for a Doha fintech."},
]
class Probe(BaseModel):
message: str
stage: str
wants_to_end: bool
def main() -> None:
load_dotenv()
client = llm._client()
for model in sys.argv[1:] or CANDIDATES:
ok, times = 0, []
for _ in range(3):
start = time.time()
try:
text = llm._complete(client, model, PROMPT, 200, json_mode=True)
Probe.model_validate(llm.extract_json(text))
ok += 1
except Exception as exc:
print(f" {model}: {type(exc).__name__}: {str(exc)[:120]}")
times.append(time.time() - start)
print(f"{model:45s} valid {ok}/3 avg {sum(times) / len(times):.1f}s")
if __name__ == "__main__":
main()
|