Hugging Face's logo

jameVee
/
ToolACE-Hallucination-Detector

Text Classification
Scikit-learn
Joblib
Safetensors
English
hallucination-detection
tool-calling
span-detection
ensemble
Model card Files
xet
 Community
ToolACE-Hallucination-Detector
File size: 1,738 Bytes
131239b
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
{
  "lexical_features": [
    "num_spans",
    "max_span_score",
    "mean_span_score",
    "span_char_fraction",
    "max_span_len"
  ],
  "lettuce_features": [
    "num_spans",
    "max_span_score",
    "mean_span_score",
    "span_char_fraction",
    "max_span_len"
  ],
  "lookback_features": [
    "mean_ratio",
    "min_ratio",
    "frac_low_03",
    "frac_low_02",
    "std_ratio",
    "bottom3_mean",
    "longest_low_streak",
    "mean_ratio_numeric",
    "frac_low_numeric",
    "num_spans",
    "max_span_score",
    "mean_span_score",
    "span_char_fraction",
    "max_span_len"
  ],
  "seed": 1241,
  "split": {
    "train": 3308,
    "test": 828
  },
  "backbone_lettuce": "KRLabsOrg/lettucedect-base-modernbert-en-v1",
  "backbone_lookback": "Qwen/Qwen2.5-0.5B",
  "source_dataset": "jameVee/ToolACE-Hallucination",
  "hallucination_types": [
    "missing_tool",
    "overgeneration",
    "tool_output_contradiction"
  ],
  "metrics_test": {
    "lexical_span_verifier": {
      "accuracy": 0.5664251207729468,
      "f1": 0.5232403718459495,
      "auroc": 0.6025060625811036,
      "span_f1": 0.052821437773769644
    },
    "lettuce_span_supervised": {
      "accuracy": 0.677536231884058,
      "f1": 0.6317241379310344,
      "auroc": 0.7214236643238388,
      "span_f1": 0.20837031474021983
    },
    "lookback_span_supervised": {
      "accuracy": 0.4830917874396135,
      "f1": 0.5532359081419624,
      "auroc": 0.5116260635205834,
      "span_f1": 0.0
    },
    "soft_vote_ensemble": {
      "accuracy": 0.6763285024154589,
      "f1": 0.6598984771573604,
      "auroc": 0.7214089849507066,
      "span_f1": 0.17918399775669488
    }
  }
}