File size: 4,260 Bytes
4f675c6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
{
 "dataset": "llamaindex/ExtractBench --test subset (6 documents: 3 short, 2 medium, 1 long)",
 "setup": "two-stage: parse stage -> Qwen2.5-3B-Instruct Q4_K_M via llama.cpp on 4 CPU cores, 12,288-token context, same prompt/budget/scorer for both arms; harness llama-extract-bench",
 "arms": {
  "docparse_extract_local": {
   "mean_value_f1": 35.8,
   "mean_value_precision": 63.8,
   "mean_value_recall": 32.5,
   "documents": [
    {
     "document": "short/W14-Atascosa SWD Well No. 4 - W-14 (Updated 01.22.2025)",
     "value_f1": 0.6457,
     "value_precision": 0.7152,
     "value_recall": 0.5885,
     "grounded_f1": 0.0,
     "text_chars_sent": 157,
     "pages_truncated": 0,
     "schema_prune_rounds": 4,
     "finish_reason": "stop"
    },
    {
     "document": "short/bianco-2024",
     "value_f1": 0.8456,
     "value_precision": 0.8693,
     "value_recall": 0.8231,
     "grounded_f1": 0.0,
     "text_chars_sent": 6344,
     "pages_truncated": 9,
     "schema_prune_rounds": 2,
     "finish_reason": "stop"
    },
    {
     "document": "short/real_wyo_Goshen_2024",
     "value_f1": 0.1958,
     "value_precision": 0.4277,
     "value_recall": 0.1269,
     "grounded_f1": 0.0,
     "text_chars_sent": 8711,
     "pages_truncated": 0,
     "schema_prune_rounds": 0,
     "finish_reason": "stop"
    },
    {
     "document": "medium/veralto_earnings_deck_q4fy25",
     "value_f1": 0.35,
     "value_precision": 0.35,
     "value_recall": 0.35,
     "grounded_f1": 0.0,
     "text_chars_sent": 8991,
     "pages_truncated": 34,
     "schema_prune_rounds": 0,
     "finish_reason": "stop"
    },
    {
     "document": "medium/real_pueblo_oct_2025",
     "value_f1": 0.0984,
     "value_precision": 0.7679,
     "value_recall": 0.0526,
     "grounded_f1": 0.0,
     "text_chars_sent": 9464,
     "pages_truncated": 9,
     "schema_prune_rounds": 0,
     "finish_reason": "length"
    },
    {
     "document": "long/real_sm0801_eco_full",
     "value_f1": 0.0139,
     "value_precision": 0.6977,
     "value_recall": 0.007,
     "grounded_f1": 0.0,
     "text_chars_sent": 19243,
     "pages_truncated": 55,
     "schema_prune_rounds": 0,
     "finish_reason": "length"
    }
   ]
  },
  "pypdf_extract_local": {
   "mean_value_f1": 32.9,
   "mean_value_precision": 66.3,
   "mean_value_recall": 29.8,
   "documents": [
    {
     "document": "short/W14-Atascosa SWD Well No. 4 - W-14 (Updated 01.22.2025)",
     "value_f1": 0.6457,
     "value_precision": 0.7152,
     "value_recall": 0.5885,
     "grounded_f1": 0.0,
     "text_chars_sent": 122,
     "pages_truncated": 0,
     "schema_prune_rounds": 4,
     "finish_reason": "stop"
    },
    {
     "document": "short/real_wyo_Goshen_2024",
     "value_f1": 0.0801,
     "value_precision": 0.6098,
     "value_recall": 0.0429,
     "grounded_f1": 0.0,
     "text_chars_sent": 4925,
     "pages_truncated": 0,
     "schema_prune_rounds": 0,
     "finish_reason": "stop"
    },
    {
     "document": "short/bianco-2024",
     "value_f1": 0.8518,
     "value_precision": 0.8796,
     "value_recall": 0.8256,
     "grounded_f1": 0.0,
     "text_chars_sent": 181,
     "pages_truncated": 0,
     "schema_prune_rounds": 2,
     "finish_reason": "stop"
    },
    {
     "document": "medium/veralto_earnings_deck_q4fy25",
     "value_f1": 0.25,
     "value_precision": 0.25,
     "value_recall": 0.25,
     "grounded_f1": 0.0,
     "text_chars_sent": 8991,
     "pages_truncated": 30,
     "schema_prune_rounds": 0,
     "finish_reason": "stop"
    },
    {
     "document": "medium/real_pueblo_oct_2025",
     "value_f1": 0.0965,
     "value_precision": 0.7342,
     "value_recall": 0.0516,
     "grounded_f1": 0.0,
     "text_chars_sent": 9464,
     "pages_truncated": 8,
     "schema_prune_rounds": 0,
     "finish_reason": "length"
    },
    {
     "document": "long/real_sm0801_eco_full",
     "value_f1": 0.0519,
     "value_precision": 0.7877,
     "value_recall": 0.0269,
     "grounded_f1": 0.0,
     "text_chars_sent": 9662,
     "pages_truncated": 53,
     "schema_prune_rounds": 0,
     "finish_reason": "length"
    }
   ]
  }
 }
}