tests/test_trace_artifacts.py

from __future__ import annotations

import json
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
SRC = ROOT / 'src'
for candidate in (ROOT, SRC):
    if str(candidate) not in sys.path:
        sys.path.insert(0, str(candidate))

from app_kit.config import load_app_config
from app_kit.demo_packs import list_demo_packs
from app_kit.eval_runner import run_eval_for_project
from app_kit.storage import SQLiteStore
from apps._base import run_pack_with_trace
from apps.p1_elder_paperwork.app import create_project_spec


def _prepare_env(monkeypatch, tmp_path: Path) -> None:
    monkeypatch.setenv('APP_ROOT_DIR', str(ROOT))
    monkeypatch.setenv('ARTIFACT_DIR', str(tmp_path / 'artifacts'))
    monkeypatch.setenv('SQLITE_PATH', str(tmp_path / 'sqlite' / 'p1.sqlite3'))


def _first_pack_path() -> Path:
    spec = create_project_spec()
    config = load_app_config(spec.key, data_subdir=spec.data_subdir)
    return list_demo_packs(config.data_dir)[0]


def test_app_load_pack_emits_trace_artifact(tmp_path, monkeypatch):
    _prepare_env(monkeypatch, tmp_path)
    spec = create_project_spec()
    config = load_app_config(spec.key, data_subdir=spec.data_subdir)
    pack_path = _first_pack_path()
    store = SQLiteStore(config.sqlite_path, config.artifact_dir)
    try:
        output, status, trace_path = run_pack_with_trace(spec, store, config, pack_path)
    finally:
        store.close()

    assert output
    assert 'trace' in status.lower()
    assert trace_path.exists()

    trace = json.loads(trace_path.read_text(encoding='utf-8'))
    assert trace['kind'] == 'app-load'
    assert trace['project'] == spec.key
    assert trace['pack_path'] == str(pack_path)
    assert trace['trace_path'] == str(trace_path)
    assert trace['timestamp']
    assert trace['inputs']
    assert 'parsed_outputs' in trace
    assert trace['model_name']


def test_eval_runner_emits_trace_artifact(tmp_path, monkeypatch):
    _prepare_env(monkeypatch, tmp_path)
    pack_path = _first_pack_path()
    result = run_eval_for_project('apps.p1_elder_paperwork.app', pack_path)

    trace_path = Path(result.trace_path)
    assert trace_path.exists()
    trace = json.loads(trace_path.read_text(encoding='utf-8'))
    assert trace['kind'] == 'eval'
    assert trace['project'] == result.project
    assert trace['pack_id'] == result.pack_id
    assert trace['trace_path'] == str(trace_path)
    assert trace['model_id']
    assert trace['adapter_name']
    assert trace['generation_stats']