File size: 8,327 Bytes
6e01804 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 | """Measured provenance coverage for FrontierLab evidence graphs.
Trace coverage is about whether research artifacts can be followed back to known
sources. It is not an accuracy, entailment, or scientific-validity metric.
"""
from __future__ import annotations
from collections import defaultdict
from .evidence_graph import SCHEMA as EVIDENCE_GRAPH_SCHEMA
IN_SCOPE_TYPES = ("claim", "hypothesis", "plan_step")
def _index(graph: dict):
nodes = {node["id"]: node for node in graph.get("nodes", []) if isinstance(node, dict)}
outgoing = defaultdict(list)
for edge in graph.get("edges", []):
if isinstance(edge, dict):
outgoing[edge.get("source")].append(edge)
return nodes, outgoing
def _headline_scope(node: dict) -> bool:
ntype = node.get("type")
if ntype == "claim":
return node.get("gate_status") == "PASSED_EVIDENCE_GATE"
return ntype in {"hypothesis", "plan_step"}
def trace_one(graph: dict, artifact_id: str) -> dict:
"""Trace one artifact to papers and evaluate strict EXTRACTED completeness."""
if not isinstance(graph, dict) or graph.get("schema") != EVIDENCE_GRAPH_SCHEMA:
raise ValueError("Expected a FrontierLab evidence graph v2")
nodes, outgoing = _index(graph)
artifact = nodes.get(artifact_id)
if artifact is None:
raise ValueError("Unknown artifact")
if artifact.get("type") not in IN_SCOPE_TYPES:
raise ValueError("Trace coverage applies to claim/hypothesis/plan_step artifacts")
papers = set()
blocking_edges = []
unresolved = []
dead_ends = []
seen = set()
stack = [artifact_id]
while stack:
node_id = stack.pop()
if node_id in seen:
continue
seen.add(node_id)
node = nodes.get(node_id)
if node is None:
dead_ends.append(node_id)
continue
if node.get("type") == "paper":
papers.add(node_id.split(":", 1)[1])
continue
unknown = node.get("unresolved_evidence_ids")
if isinstance(unknown, list) and unknown:
unresolved.extend(str(value) for value in unknown)
edges = outgoing.get(node_id, [])
if not edges:
dead_ends.append(node_id)
continue
for edge in edges:
if edge.get("provenance") != "EXTRACTED":
blocking_edges.append({
"source": edge.get("source"),
"target": edge.get("target"),
"relation": edge.get("relation"),
"provenance": edge.get("provenance"),
"reason": edge.get("reason"),
})
target = edge.get("target")
if isinstance(target, str):
stack.append(target)
else:
dead_ends.append(str(target))
source_traceable = bool(papers)
strict_extracted = bool(
papers and not blocking_edges and not unresolved and not dead_ends
)
return {
"artifact_id": artifact_id,
"artifact_type": artifact.get("type"),
"source_traceable": source_traceable,
"strict_extracted": strict_extracted,
"papers": sorted(papers),
"blocking_edges": blocking_edges,
"unresolved_evidence_ids": sorted(set(unresolved)),
"dead_ends": sorted(set(dead_ends)),
}
def trace_coverage(graph: dict) -> dict:
"""Calculate measured trace coverage over downstream-eligible artifacts.
Headline denominator:
* claims that passed the evidence gate;
* all hypotheses;
* all plan steps.
Rejected claim nodes remain visible as exclusions rather than silently
lowering or inflating the headline denominator.
"""
if not isinstance(graph, dict) or graph.get("schema") != EVIDENCE_GRAPH_SCHEMA:
raise ValueError("Expected a FrontierLab evidence graph v2")
nodes = [node for node in graph.get("nodes", []) if isinstance(node, dict)]
scoped = [node for node in nodes if _headline_scope(node)]
records = [trace_one(graph, node["id"]) for node in scoped]
total = len(records)
source_traced = sum(record["source_traceable"] for record in records)
strict = sum(record["strict_extracted"] for record in records)
by_type = {}
for ntype in IN_SCOPE_TYPES:
subset = [record for record in records if record["artifact_type"] == ntype]
denom = len(subset)
by_type[ntype] = {
"artifacts": denom,
"source_traceable": sum(record["source_traceable"] for record in subset),
"strict_extracted": sum(record["strict_extracted"] for record in subset),
"source_trace_coverage": round(
sum(record["source_traceable"] for record in subset) / denom, 4
) if denom else None,
"strict_extracted_coverage": round(
sum(record["strict_extracted"] for record in subset) / denom, 4
) if denom else None,
}
rejected_claims = sum(
node.get("type") == "claim"
and node.get("gate_status") == "REJECTED_BY_EVIDENCE_GATE"
for node in nodes
)
blocking_counts = {"INFERRED": 0, "AMBIGUOUS": 0, "OTHER": 0}
for record in records:
for edge in record["blocking_edges"]:
provenance = edge.get("provenance")
if provenance in blocking_counts:
blocking_counts[provenance] += 1
else:
blocking_counts["OTHER"] += 1
return {
"schema": "frontierlab.trace-coverage.v1",
"scope_definition": (
"Claims that passed the evidence gate plus all hypotheses and plan steps."
),
"material_artifacts": total,
"source_traceable_artifacts": source_traced,
"strict_extracted_artifacts": strict,
"source_untraceable_artifacts": total - source_traced,
"strict_blocked_artifacts": total - strict,
"blocking_edges_by_provenance": blocking_counts,
"source_trace_coverage": round(source_traced / total, 4) if total else None,
"strict_extracted_coverage": round(strict / total, 4) if total else None,
"excluded_rejected_claims": rejected_claims,
"unattached_rejected_claims": int(graph.get("unattached_rejected_claims", 0) or 0),
"by_type": by_type,
"artifacts": records,
"scientific_accuracy_measured": False,
"semantic_entailment_verified": False,
"note": (
"Source Trace Coverage measures whether an in-scope artifact reaches at least "
"one known Paper. Strict Extracted Coverage additionally requires every reachable "
"provenance edge to be EXTRACTED, no unresolved source IDs, and no dead ends. "
"Neither metric proves that a scientific claim is true."
),
}
def source_audit_summary(graph: dict) -> dict:
if not isinstance(graph, dict) or graph.get("schema") != EVIDENCE_GRAPH_SCHEMA:
raise ValueError("Expected a FrontierLab evidence graph v2")
nodes = [node for node in graph.get("nodes", []) if isinstance(node, dict)]
edges = [edge for edge in graph.get("edges", []) if isinstance(edge, dict)]
passed_claims = [
node for node in nodes
if node.get("type") == "claim"
and node.get("gate_status") == "PASSED_EVIDENCE_GATE"
]
anchored_claim_ids = {
edge.get("source")
for edge in edges
if edge.get("source", "").startswith("claim:")
and edge.get("provenance") == "EXTRACTED"
and edge.get("relation") in {"SUPPORTS", "CONTRADICTS"}
}
return {
"papers": sum(node.get("type") == "paper" for node in nodes),
"passages": sum(node.get("type") == "passage" for node in nodes),
"passed_claims": len(passed_claims),
"anchored_passed_claims": sum(
node.get("id") in anchored_claim_ids for node in passed_claims
),
"rejected_claims_attached": sum(
node.get("type") == "claim"
and node.get("gate_status") == "REJECTED_BY_EVIDENCE_GATE"
for node in nodes
),
"rejected_claims_unattached": int(graph.get("unattached_rejected_claims", 0) or 0),
"source_audit_is_semantic_entailment": False,
}
|