Add TOTEM token audit and Groq provider support

.gitignore

@@ -0,0 +1,4 @@
+.DS_Store
+__pycache__/
+*.py[cod]
+data/token_audit/

app.py

@@ -3175,7 +3175,7 @@ def run_totem_analysis_from_context(active_path: str, manuscript_context_json: s
     Full bridge run:
     - workbook matrix is source of truth
     - manuscript context is evidence
-    - hidden OpenAI skill returns strict JSON
+    - hidden LLM skill returns strict JSON
     - validated JSON updates dashboard
     """
     if not manuscript_context_json:
@@ -3213,7 +3213,9 @@ def run_totem_analysis_from_context(active_path: str, manuscript_context_json: s
         debug_log = {
             "status": "ok",
             "gate_summary": gate_summary,
+            "provider": debug.get("provider"),
             "model": debug.get("model"),
+            "api_key_env": debug.get("api_key_env"),
             "manuscript_words": debug.get("word_count"),
             "manuscript_path": ctx.path,
             "rubric_metrics": len(rubric.get("metrics", [])),
@@ -3221,6 +3223,9 @@ def run_totem_analysis_from_context(active_path: str, manuscript_context_json: s
             "skill_path": debug.get("skill_path"),
             "schema_pass": debug.get("schema_pass"),
             "score_fields": list(SCORE_FIELDS),
+            "actual_tokens": debug.get("actual_tokens"),
+            "token_audit_path": debug.get("token_audit_path"),
+            "token_audit_transaction_id": debug.get("token_audit_transaction_id"),
         }
         return dashboard_html, log_df, summary, json.dumps(debug_log, indent=2)
     except Exception as exc:

src/token_meter.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+import os
+import time
+import uuid
+from pathlib import Path
+from typing import Any
+
+
+APP_ROOT = Path(__file__).resolve().parents[1]
+DEFAULT_AUDIT_DIR = APP_ROOT / "data" / "token_audit"
+
+
+def new_transaction_id(prefix: str = "totem") -> str:
+    stamp = time.strftime("%Y%m%d-%H%M%S", time.gmtime())
+    return f"{prefix}_{stamp}_{uuid.uuid4().hex[:10]}"
+
+
+def stable_hash(value: Any) -> str:
+    if isinstance(value, str):
+        payload = value
+    else:
+        payload = json.dumps(value, ensure_ascii=False, sort_keys=True, default=str)
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def compact_json(value: Any) -> str:
+    return json.dumps(value, ensure_ascii=False, sort_keys=True, separators=(",", ":"), default=str)
+
+
+def estimate_text_tokens(text: str, model: str | None = None) -> dict[str, Any]:
+    """
+    Estimate tokens before an API call. Exact transaction totals should still come
+    from the provider response usage object after the call completes.
+    """
+    text = text or ""
+    try:
+        import tiktoken  # type: ignore
+
+        try:
+            enc = tiktoken.encoding_for_model(model or "")
+            method = f"tiktoken:{enc.name}:model"
+        except Exception:
+            enc = tiktoken.get_encoding("cl100k_base")
+            method = "tiktoken:cl100k_base:fallback"
+        return {
+            "tokens": len(enc.encode(text)),
+            "chars": len(text),
+            "method": method,
+        }
+    except Exception:
+        # Conservative no-dependency approximation. Most English prompt material
+        # lands around 3.5-4 chars/token, but JSON punctuation pushes lower.
+        return {
+            "tokens": int(math.ceil(len(text) / 4)) if text else 0,
+            "chars": len(text),
+            "method": "heuristic:ceil(chars/4)",
+        }
+
+
+def estimate_components(components: dict[str, str], model: str | None = None) -> dict[str, Any]:
+    by_component = {
+        name: estimate_text_tokens(text, model=model)
+        for name, text in components.items()
+    }
+    return {
+        "components": by_component,
+        "estimated_input_total": sum(item["tokens"] for item in by_component.values()),
+        "methods": sorted({item["method"] for item in by_component.values()}),
+    }
+
+
+def extract_response_usage(response: Any) -> dict[str, int | None]:
+    usage = getattr(response, "usage", None)
+    if usage is None and isinstance(response, dict):
+        usage = response.get("usage")
+    if usage is None:
+        return {
+            "prompt_tokens": None,
+            "completion_tokens": None,
+            "total_tokens": None,
+        }
+
+    def get(name: str) -> int | None:
+        value = getattr(usage, name, None)
+        if value is None and isinstance(usage, dict):
+            value = usage.get(name)
+        try:
+            return int(value) if value is not None else None
+        except Exception:
+            return None
+
+    return {
+        "prompt_tokens": get("prompt_tokens"),
+        "completion_tokens": get("completion_tokens"),
+        "total_tokens": get("total_tokens"),
+    }
+
+
+def audit_dir() -> Path:
+    return Path(os.getenv("TOTEM_TOKEN_AUDIT_DIR") or DEFAULT_AUDIT_DIR)
+
+
+def append_token_audit(record: dict[str, Any]) -> str:
+    out_dir = audit_dir()
+    out_dir.mkdir(parents=True, exist_ok=True)
+    path = out_dir / "totem_token_usage.jsonl"
+    with path.open("a", encoding="utf-8") as fh:
+        fh.write(compact_json(record) + "\n")
+    return str(path)

src/totem_bridge.py

@@ -2,6 +2,7 @@ from __future__ import annotations
 
 import json
 import os
+import time
 from dataclasses import dataclass
 from pathlib import Path
 from typing import Any
@@ -9,6 +10,15 @@ from typing import Any
 import pandas as pd
 
 from src.codex_extractor import clean_text, extract_text_from_file
+from src.token_meter import (
+    append_token_audit,
+    compact_json,
+    estimate_components,
+    estimate_text_tokens,
+    extract_response_usage,
+    new_transaction_id,
+    stable_hash,
+)
 from src.totem_workbook import METRICS, protocol_table, protocol_weights
 
 
@@ -54,6 +64,15 @@ class ManuscriptContext:
     page_trace: list[dict[str, Any]]
 
 
+@dataclass
+class LLMProviderConfig:
+    provider: str
+    model: str
+    api_key: str
+    api_key_env: str
+    base_url: str | None = None
+
+
 def extract_workbook_matrix(path: Path) -> dict[str, Any]:
     if not path.exists():
         raise BridgeError(f"Workbook not found: {path}")
@@ -142,29 +161,8 @@ def run_totem_skill(
     manuscript: ManuscriptContext,
 ) -> tuple[dict[str, Any], dict[str, Any]]:
     skill_text, skill_path = load_totem_skill()
-
-    if _env_truthy("TOTEM_FAKE_API"):
-        if os.getenv("SPACE_ID") and not _env_truthy("TOTEM_ALLOW_FAKE_API_IN_SPACE"):
-            raise BridgeError("TOTEM_FAKE_API is disabled on Hugging Face Spaces.")
-        raw_payload = _fake_dashboard_payload(manuscript)
-        validated = validate_dashboard_payload(raw_payload)
-        return validated, {
-            "model": "fake-audit",
-            "word_count": manuscript.word_count,
-            "raw_response_chars": len(json.dumps(raw_payload)),
-            "skill_path": skill_path,
-            "skill_loaded": True,
-            "schema_pass": True,
-        }
-
-    api_key = os.getenv("OPENAI_API_KEY")
-    if not api_key:
-        raise BridgeError("OPENAI_API_KEY is not configured.")
-
-    from openai import OpenAI
-
-    model = os.getenv("TOTEM_OPENAI_MODEL", "gpt-4.1-mini")
-    client = OpenAI(api_key=api_key)
+    transaction_id = new_transaction_id("totem")
+    contract = dashboard_output_contract()
 
     system_prompt = f"""
 You are the hidden TOTEM Analysis skill inside TOTEM Studio.
@@ -182,41 +180,340 @@ Use the workbook matrix as scoring authority and manuscript text as evidence.
         "rubric_matrix": rubric_matrix,
         "manuscript_cleaned_text": manuscript.cleaned_text,
         "scoring_dimensions": SCORE_LABELS,
-        "required_output_contract": dashboard_output_contract(),
+        "required_output_contract": contract,
         "instruction": (
             "Return exactly one JSON object matching required_output_contract. "
             "Scores must be 0..100 dashboard values for the six dimensions. "
             "Do not write a report and do not include manuscript excerpts."
         ),
     }
+    user_payload_json = json.dumps(user_payload, ensure_ascii=False)
 
-    resp = client.chat.completions.create(
-        model=model,
-        response_format={"type": "json_object"},
-        temperature=0.2,
-        messages=[
-            {"role": "system", "content": system_prompt},
-            {"role": "user", "content": json.dumps(user_payload)},
-        ],
-    )
+    if _env_truthy("TOTEM_FAKE_API"):
+        if os.getenv("SPACE_ID") and not _env_truthy("TOTEM_ALLOW_FAKE_API_IN_SPACE"):
+            raise BridgeError("TOTEM_FAKE_API is disabled on Hugging Face Spaces.")
+        raw_payload = _fake_dashboard_payload(manuscript)
+        validated = validate_dashboard_payload(raw_payload)
+        raw_payload_json = json.dumps(raw_payload, ensure_ascii=False)
+        audit_path = _write_token_audit(
+            transaction_id=transaction_id,
+            status="ok",
+            provider="fake",
+            model="fake-audit",
+            api_key_env=None,
+            skill_path=skill_path,
+            skill_text=skill_text,
+            rubric_matrix=rubric_matrix,
+            manuscript=manuscript,
+            contract=contract,
+            system_prompt=system_prompt,
+            user_payload_json=user_payload_json,
+            actual_tokens=None,
+            raw_response=raw_payload_json,
+            latency_ms=0,
+            error=None,
+        )
+        return validated, {
+            "provider": "fake",
+            "model": "fake-audit",
+            "word_count": manuscript.word_count,
+            "raw_response_chars": len(raw_payload_json),
+            "skill_path": skill_path,
+            "skill_loaded": True,
+            "schema_pass": True,
+            "token_audit_path": audit_path,
+            "token_audit_transaction_id": transaction_id,
+        }
+
+    provider_config = _select_llm_provider()
+
+    from openai import OpenAI
+
+    client_kwargs: dict[str, Any] = {"api_key": provider_config.api_key}
+    if provider_config.base_url:
+        client_kwargs["base_url"] = provider_config.base_url
+    client = OpenAI(**client_kwargs)
+
+    started = time.perf_counter()
+    resp = None
+    raw = ""
+    try:
+        resp = client.chat.completions.create(
+            model=provider_config.model,
+            response_format={"type": "json_object"},
+            temperature=0.2,
+            messages=[
+                {"role": "system", "content": system_prompt},
+                {"role": "user", "content": user_payload_json},
+            ],
+        )
+    except Exception as exc:
+        latency_ms = int(round((time.perf_counter() - started) * 1000))
+        audit_path = _write_token_audit(
+            transaction_id=transaction_id,
+            status="api_error",
+            provider=provider_config.provider,
+            model=provider_config.model,
+            api_key_env=provider_config.api_key_env,
+            skill_path=skill_path,
+            skill_text=skill_text,
+            rubric_matrix=rubric_matrix,
+            manuscript=manuscript,
+            contract=contract,
+            system_prompt=system_prompt,
+            user_payload_json=user_payload_json,
+            actual_tokens=None,
+            raw_response="",
+            latency_ms=latency_ms,
+            error=f"{type(exc).__name__}: {exc}",
+        )
+        raise BridgeError(
+            f"{provider_config.provider} API call failed. Token audit: {audit_path}. {type(exc).__name__}: {exc}"
+        ) from exc
+
+    latency_ms = int(round((time.perf_counter() - started) * 1000))
+    actual_tokens = extract_response_usage(resp)
     raw = (resp.choices[0].message.content or "").strip()
     try:
         parsed = json.loads(raw)
     except Exception as exc:
+        audit_path = _write_token_audit(
+            transaction_id=transaction_id,
+            status="invalid_json",
+            provider=provider_config.provider,
+            model=provider_config.model,
+            api_key_env=provider_config.api_key_env,
+            skill_path=skill_path,
+            skill_text=skill_text,
+            rubric_matrix=rubric_matrix,
+            manuscript=manuscript,
+            contract=contract,
+            system_prompt=system_prompt,
+            user_payload_json=user_payload_json,
+            actual_tokens=actual_tokens,
+            raw_response=raw,
+            latency_ms=latency_ms,
+            error=f"{type(exc).__name__}: {exc}",
+        )
         raise BridgeValidationError(f"Model returned invalid JSON: {exc}") from exc
 
-    validated = validate_dashboard_payload(parsed)
+    try:
+        validated = validate_dashboard_payload(parsed)
+    except BridgeValidationError as exc:
+        audit_path = _write_token_audit(
+            transaction_id=transaction_id,
+            status="schema_error",
+            provider=provider_config.provider,
+            model=provider_config.model,
+            api_key_env=provider_config.api_key_env,
+            skill_path=skill_path,
+            skill_text=skill_text,
+            rubric_matrix=rubric_matrix,
+            manuscript=manuscript,
+            contract=contract,
+            system_prompt=system_prompt,
+            user_payload_json=user_payload_json,
+            actual_tokens=actual_tokens,
+            raw_response=raw,
+            latency_ms=latency_ms,
+            error=f"{type(exc).__name__}: {exc}",
+        )
+        raise BridgeValidationError(f"{exc}. Token audit: {audit_path}") from exc
+
+    audit_path = _write_token_audit(
+        transaction_id=transaction_id,
+        status="ok",
+        provider=provider_config.provider,
+        model=provider_config.model,
+        api_key_env=provider_config.api_key_env,
+        skill_path=skill_path,
+        skill_text=skill_text,
+        rubric_matrix=rubric_matrix,
+        manuscript=manuscript,
+        contract=contract,
+        system_prompt=system_prompt,
+        user_payload_json=user_payload_json,
+        actual_tokens=actual_tokens,
+        raw_response=raw,
+        latency_ms=latency_ms,
+        error=None,
+    )
     debug = {
-        "model": model,
+        "provider": provider_config.provider,
+        "model": provider_config.model,
+        "api_key_env": provider_config.api_key_env,
         "word_count": manuscript.word_count,
         "raw_response_chars": len(raw),
         "skill_path": skill_path,
         "skill_loaded": True,
         "schema_pass": True,
+        "actual_tokens": actual_tokens,
+        "token_audit_path": audit_path,
+        "token_audit_transaction_id": transaction_id,
     }
     return validated, debug
 
 
+def _select_llm_provider() -> LLMProviderConfig:
+    # Hugging Face exposes secrets exactly as named. Accept Jamal's early
+    # `grok_key` spelling, but normalize internally to the standard Groq name.
+    if os.environ.get("grok_key") and not os.environ.get("GROQ_API_KEY"):
+        os.environ["GROQ_API_KEY"] = os.environ["grok_key"]
+
+    requested = str(os.getenv("TOTEM_LLM_PROVIDER") or "").strip().lower()
+    if requested == "grok":
+        requested = "groq"
+
+    groq_key, groq_env = _first_env_value(
+        (
+            "GROQ_API_KEY",
+            "groq_key",
+            "GROQ_KEY",
+            "GROK_API_KEY",
+            "GROK_KEY",
+        )
+    )
+    openai_key, openai_env = _first_env_value(("OPENAI_API_KEY",))
+
+    if not requested:
+        requested = "groq" if groq_key else "openai"
+
+    if requested == "groq":
+        if not groq_key:
+            raise BridgeError("Groq is selected but no GROQ_API_KEY/groq_key secret is configured.")
+        return LLMProviderConfig(
+            provider="groq",
+            model=os.getenv("TOTEM_GROQ_MODEL") or os.getenv("GROQ_MODEL") or "llama-3.3-70b-versatile",
+            api_key=groq_key,
+            api_key_env=groq_env or "GROQ_API_KEY",
+            base_url=os.getenv("GROQ_BASE_URL") or "https://api.groq.com/openai/v1",
+        )
+
+    if requested == "openai":
+        if not openai_key and groq_key:
+            return LLMProviderConfig(
+                provider="groq",
+                model=os.getenv("TOTEM_GROQ_MODEL") or os.getenv("GROQ_MODEL") or "llama-3.3-70b-versatile",
+                api_key=groq_key,
+                api_key_env=groq_env or "GROQ_API_KEY",
+                base_url=os.getenv("GROQ_BASE_URL") or "https://api.groq.com/openai/v1",
+            )
+        if not openai_key:
+            raise BridgeError("OpenAI is selected but OPENAI_API_KEY is not configured.")
+        return LLMProviderConfig(
+            provider="openai",
+            model=os.getenv("TOTEM_OPENAI_MODEL", "gpt-4.1-mini"),
+            api_key=openai_key,
+            api_key_env=openai_env or "OPENAI_API_KEY",
+            base_url=os.getenv("OPENAI_BASE_URL") or None,
+        )
+
+    raise BridgeError("TOTEM_LLM_PROVIDER must be 'groq' or 'openai'.")
+
+
+def _write_token_audit(
+    *,
+    transaction_id: str,
+    status: str,
+    provider: str,
+    model: str,
+    api_key_env: str | None,
+    skill_path: str,
+    skill_text: str,
+    rubric_matrix: dict[str, Any],
+    manuscript: ManuscriptContext,
+    contract: dict[str, Any],
+    system_prompt: str,
+    user_payload_json: str,
+    actual_tokens: dict[str, int | None] | None,
+    raw_response: str,
+    latency_ms: int,
+    error: str | None,
+) -> str:
+    rubric_json = compact_json(rubric_matrix)
+    contract_json = compact_json(contract)
+    user_wrapper_json = compact_json(
+        {
+            "task": "score_manuscript_against_workbook_matrix_for_dashboard",
+            "scoring_dimensions": SCORE_LABELS,
+            "instruction": "Return exactly one JSON object matching required_output_contract.",
+        }
+    )
+    system_wrapper = """
+You are the hidden TOTEM Analysis skill inside TOTEM Studio.
+Use the skill instructions as the scoring method, but do not create a DOCX report for this dashboard run.
+Return strict JSON only. No prose. No markdown. No code fences.
+Use the workbook matrix as scoring authority and manuscript text as evidence.
+""".strip()
+
+    component_estimates = estimate_components(
+        {
+            "system_wrapper": system_wrapper,
+            "skill_md": skill_text,
+            "workbook_rubric_json": rubric_json,
+            "manuscript_cleaned_text": manuscript.cleaned_text,
+            "output_contract_json": contract_json,
+            "user_wrapper_json": user_wrapper_json,
+        },
+        model=model,
+    )
+    system_wire = estimate_text_tokens(system_prompt, model=model)
+    user_wire = estimate_text_tokens(user_payload_json, model=model)
+
+    record = {
+        "transaction_id": transaction_id,
+        "created_at_utc": time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()),
+        "status": status,
+        "provider": provider,
+        "model": model,
+        "api_key_env": api_key_env,
+        "latency_ms": latency_ms,
+        "skill": {
+            "path": skill_path,
+            "sha256": stable_hash(skill_text),
+            "chars": len(skill_text),
+        },
+        "workbook": {
+            "sha256": stable_hash(rubric_matrix),
+            "metric_count": len(rubric_matrix.get("metrics", [])) if isinstance(rubric_matrix, dict) else 0,
+        },
+        "manuscript": {
+            "path": manuscript.path,
+            "sha256": stable_hash(manuscript.cleaned_text),
+            "chars": len(manuscript.cleaned_text),
+            "word_count": manuscript.word_count,
+        },
+        "estimated_tokens": {
+            **component_estimates,
+            "wire_prompt": {
+                "system_prompt": system_wire,
+                "user_message": user_wire,
+                "estimated_prompt_total": system_wire["tokens"] + user_wire["tokens"],
+            },
+        },
+        "actual_tokens": actual_tokens
+        or {
+            "prompt_tokens": None,
+            "completion_tokens": None,
+            "total_tokens": None,
+        },
+        "response": {
+            "sha256": stable_hash(raw_response) if raw_response else None,
+            "chars": len(raw_response or ""),
+        },
+        "error": error,
+    }
+    return append_token_audit(record)
+
+
+def _first_env_value(names: tuple[str, ...]) -> tuple[str | None, str | None]:
+    for name in names:
+        value = os.getenv(name)
+        if value:
+            return value, name
+    return None, None
+
+
 def validate_dashboard_payload(payload: dict[str, Any]) -> dict[str, Any]:
     if not isinstance(payload, dict):
         raise BridgeValidationError("Dashboard payload must be a JSON object.")