File size: 1,596 Bytes
0d1866c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 | from __future__ import annotations
from dataclasses import dataclass
from datetime import datetime, timezone
from pathlib import Path
import json
import sys
REPO_ROOT = Path(__file__).resolve().parents[2]
if str(REPO_ROOT) not in sys.path:
sys.path.insert(0, str(REPO_ROOT))
from repo_paths import EVAL_DIR # noqa: E402
LOCAL_INTELLIGENCE_DIR = EVAL_DIR / "local_intelligence"
CARDS_PATH = LOCAL_INTELLIGENCE_DIR / "cards.json"
RESULTS_DIR = LOCAL_INTELLIGENCE_DIR / "results"
NOTEBOOK_PATH = LOCAL_INTELLIGENCE_DIR / "notebook.ipynb"
EXPORT_DIR = REPO_ROOT / "docs" / "evals" / "local-intelligence" / "v1"
JSONL_EXPORT_PATH = EXPORT_DIR / "hf_local_intelligence_eval_v1.jsonl"
MANIFEST_EXPORT_PATH = EXPORT_DIR / "hf_local_intelligence_eval_manifest_v1.json"
DEFAULT_TIMEOUT_SECONDS = 90
DEFAULT_OLLAMA_URL = "http://127.0.0.1:11434"
@dataclass(frozen=True)
class EvalCard:
payload: dict
@property
def id(self) -> str:
return self.payload["id"]
@property
def tier(self) -> str:
return self.payload["tier"]
@property
def title(self) -> str:
return self.payload["title"]
@property
def test_input(self) -> dict:
return self.payload["testCase"]["input"]
@property
def assertions(self) -> list[str]:
return self.payload["testCase"]["assertions"]
def load_cards(cards_path: Path = CARDS_PATH) -> list[EvalCard]:
data = json.loads(cards_path.read_text())
return [EvalCard(item) for item in data]
def utc_timestamp() -> str:
return datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%SZ")
|