tests/test_mvp_smoke.py

from __future__ import annotations

import json
import os
import sys
from pathlib import Path

import pytest
from jsonschema import Draft202012Validator

from gcmd_classifier.articles import load_articles
from gcmd_classifier.config import ModelSettings
from gcmd_classifier.llm import FakeModelClient
from gcmd_classifier.llm.openai_provider import OpenAIModelClient
from gcmd_classifier.models import (
    ArticleClassificationOutcome,
    ArticleLoadResult,
    ArticleProcessingStatus,
)
from gcmd_classifier.persistence import ArticleResultCache, JsonResultStore
from gcmd_classifier.pipeline import run_batch
from gcmd_classifier.vocabulary import VocabularyIndex, load_vocabulary

FULL_HIERARCHY_PATH = Path("data/gcmd_hierarchy.json")
FULL_ARTICLES_PATH = Path("data/articles.json")
PROTOTYPE_PATH = Path("prototype/app_hf_poc.py")
CLASSIFICATION_SCHEMA_PATH = Path("schemas/classification_result.schema.json")
RUN_SUMMARY_SCHEMA_PATH = Path("schemas/run_summary.schema.json")


def _decision(candidate_id: str) -> dict:
    return {
        "candidate_id": candidate_id,
        "confidence": 0.9,
        "evidence": f"Fake smoke evidence for {candidate_id}.",
        "support_type": "explicit",
        "reason": f"Fake smoke selected {candidate_id}.",
    }


def _topic_response(candidate_id: str) -> dict:
    return {"selected": [_decision(candidate_id)]}


def _term_response(candidate_id: str) -> dict:
    return {"selected": [_decision(candidate_id)], "stop_at_parent": False}


def _variable_response(candidate_id: str = "variable_0001") -> dict:
    return {"selected": [_decision(candidate_id)], "stop_at_parent": False}


def _no_topic_response() -> dict:
    return {
        "selected": [],
        "no_selection_reason": "Fake smoke no-classification article.",
    }


def _fake_smoke_actions(index: VocabularyIndex) -> list[dict]:
    topic_position, topic, term_position, term = _first_term_with_variables(index)
    actions = [
        _topic_response(f"topic_{topic_position:04d}"),
        _term_response(f"term_{term_position:04d}"),
    ]
    parent = term
    while parent.level != "Variable_Level_3":
        children = index.variables_for_parent(parent.UUID)
        if not children:
            break
        actions.append(_variable_response("variable_0001"))
        parent = children[0]
    actions.append(_no_topic_response())
    return actions


def _first_term_with_variables(index: VocabularyIndex):
    for topic_position, topic in enumerate(index.topics(), start=1):
        for term_position, term in enumerate(index.terms_for_topic(topic.UUID), start=1):
            if index.variables_for_parent(term.UUID):
                return topic_position, topic, term_position, term
    raise AssertionError("Current vocabulary did not contain a Term with Variable children.")


def _smoke_article_load_result(full_load: ArticleLoadResult) -> ArticleLoadResult:
    return full_load.model_copy(update={"articles": full_load.articles[:2]})


def test_fake_model_mvp_smoke_run_end_to_end(tmp_path: Path) -> None:
    hierarchy_before = FULL_HIERARCHY_PATH.read_bytes()
    articles_before = FULL_ARTICLES_PATH.read_bytes()
    prototype_before = PROTOTYPE_PATH.read_bytes()
    classification_schema = json.loads(CLASSIFICATION_SCHEMA_PATH.read_text())
    summary_schema = json.loads(RUN_SUMMARY_SCHEMA_PATH.read_text())

    vocabulary = load_vocabulary(FULL_HIERARCHY_PATH)
    raw_records = json.loads(FULL_ARTICLES_PATH.read_text())
    full_load = load_articles(FULL_ARTICLES_PATH)
    smoke_load = _smoke_article_load_result(full_load)
    cache = ArticleResultCache(tmp_path / "cache")
    store = JsonResultStore(tmp_path / "results")

    batch = run_batch(
        article_load_result=smoke_load,
        vocabulary=vocabulary,
        model_client=FakeModelClient(_fake_smoke_actions(vocabulary)),
        settings=ModelSettings(),
        store=store,
        cache=cache,
        run_id="fake-smoke",
        relevant_config={"smoke_subset_size": len(smoke_load.articles)},
    )

    assert len(vocabulary) == 3535
    assert isinstance(raw_records, list)
    assert full_load.source_count == len(raw_records)
    assert len(full_load.articles) == len(raw_records) - len(full_load.errors)
    assert len({article.DOI for article in full_load.articles}) == len(full_load.articles)
    if full_load.errors:
        assert all(error.code and error.message for error in full_load.errors)
        assert all(
            error.index is None or 0 <= error.index < len(raw_records) for error in full_load.errors
        )
    assert batch.summary.articles_received == len(raw_records)
    assert batch.summary.invalid_source_records == len(full_load.errors)
    assert batch.summary.processed_articles == 2
    assert batch.summary.cache_misses == 2
    assert batch.summary.cache_hits == 0
    assert batch.summary.total_model_calls and batch.summary.total_model_calls > 0
    assert batch.summary.duration_seconds is not None
    assert batch.summary.started_at is not None
    assert batch.summary.completed_at is not None

    classified = [
        result
        for result in batch.results
        if result.classification_outcome is ArticleClassificationOutcome.CLASSIFIED
    ]
    not_classified = [
        result
        for result in batch.results
        if result.classification_outcome is ArticleClassificationOutcome.NOT_CLASSIFIED
    ]
    assert classified
    assert not_classified
    assert not_classified[0].processing_status is ArticleProcessingStatus.COMPLETED
    assert not_classified[0].classifications == ()
    assert not_classified[0].no_classification_reason

    accepted = tuple(record for result in batch.results for record in result.classifications)
    assert accepted
    assert batch.summary.accepted_classifications == len(accepted)
    assert all(record.deterministic_validation.valid for record in accepted)
    assert not any(
        record.final_status == "accepted" and not record.deterministic_validation.valid
        for record in accepted
    )

    article_validator = Draft202012Validator(classification_schema)
    for result in batch.results:
        article_validator.validate(result.model_dump(mode="json"))
    Draft202012Validator(summary_schema).validate(batch.summary.model_dump(mode="json"))
    assert store.consolidated_path.exists()
    persisted = json.loads(store.consolidated_path.read_text())
    assert persisted["summary"]["run_id"] == "fake-smoke"
    assert len(persisted["articles"]) == 2

    assert "prototype.app_hf_poc" not in sys.modules
    assert FULL_HIERARCHY_PATH.read_bytes() == hierarchy_before
    assert FULL_ARTICLES_PATH.read_bytes() == articles_before
    assert PROTOTYPE_PATH.read_bytes() == prototype_before


@pytest.mark.integration
def test_optional_live_model_smoke_is_skipped_unless_explicitly_enabled(tmp_path: Path) -> None:
    if os.environ.get("GCMD_RUN_LIVE_INTEGRATION") != "1":
        pytest.skip("Set GCMD_RUN_LIVE_INTEGRATION=1 to enable live model smoke checks.")

    settings = ModelSettings.from_environment()
    if settings.provider != "openai":
        pytest.skip("Set MODEL_PROVIDER=openai for live model smoke checks.")
    if not os.environ.get(settings.api_key_env_var):
        pytest.skip(f"Set {settings.api_key_env_var} for live model smoke checks.")

    vocabulary = load_vocabulary(FULL_HIERARCHY_PATH)
    full_load = load_articles(FULL_ARTICLES_PATH)
    smoke_load = full_load.model_copy(update={"articles": full_load.articles[:1]})
    batch = run_batch(
        article_load_result=smoke_load,
        vocabulary=vocabulary,
        model_client=OpenAIModelClient(settings),
        settings=settings,
        store=JsonResultStore(tmp_path / "live-results"),
        run_id="live-smoke",
    )

    accepted = tuple(record for result in batch.results for record in result.classifications)
    assert all(record.deterministic_validation.valid for record in accepted)
    assert batch.summary.processed_articles == 1