hoytshao commited on
Commit
e7a0fbc
·
verified ·
1 Parent(s): 5a4a8c4

Update app.py

Browse files

1. Always run header and footer band OCR
Header band: always OCR from top of page down to he (and if he <= 0, use he = DEFAULT_ZONE_FRAC).
Footer band: always OCR from fs to bottom (and if fs >= 1.0, use fs = 1.0 - DEFAULT_ZONE_FRAC).
So we always run both bands and no longer skip them when the API returns no or full-page regions.
2. Larger default bands
DEFAULT_ZONE_FRAC increased from 0.08 to 0.12 (top 12% and bottom 12%) so header/footer are more likely to be covered. You can override with the GLMOCR_DEFAULT_ZONE_FRAC env var (e.g. 0.15 for 15%).
3. Config and formatter writes are best-effort
The config and result_formatter.py patches are wrapped in try/except. If the filesystem is read-only (e.g. on HF), the app still starts; for MaaS, header/footer now come from the client-side band OCR anyway

Files changed (1) hide show
  1. app.py +63 -54
app.py CHANGED
@@ -14,59 +14,63 @@ GLMOCR_BASE = os.path.dirname(glmocr.__file__)
14
  CONFIG_PATH = os.path.join(GLMOCR_BASE, "config.yaml")
15
  FORMATTER_PATH = os.path.join(GLMOCR_BASE, "postprocess", "result_formatter.py")
16
 
17
- DEFAULT_ZONE_FRAC = float(os.environ.get("GLMOCR_DEFAULT_ZONE_FRAC", "0.08"))
 
 
18
 
19
  # ---------------------------------------------------------------------------
20
- # 1. Config: include headers/footers (remove from abandon, add to text mapping)
21
  # ---------------------------------------------------------------------------
22
- with open(CONFIG_PATH, "r") as f:
23
- config = yaml.safe_load(f)
24
-
25
- config["pipeline"]["maas"]["enabled"] = True
26
- config["pipeline"]["maas"]["api_key"] = "4570c28bdea5493c9efae9dae68edc66.sGbA9DLlcX1GlvqV"
27
-
28
- to_include_as_text = {"header", "footer"}
29
- layout_section = config.get("pipeline", {}).get("layout", {})
30
- if "label_task_mapping" in layout_section:
31
- mapping = layout_section["label_task_mapping"]
32
- # Remove header/footer from abandon so they are not discarded
33
- abandon = mapping.get("abandon")
34
- if isinstance(abandon, list):
35
- mapping["abandon"] = [x for x in abandon if x not in to_include_as_text]
36
- # Add header/footer to text so they are OCR'd and output as text
37
- text_labels = mapping.get("text")
38
- if isinstance(text_labels, list):
39
- for label in to_include_as_text:
40
- if label not in text_labels:
41
- text_labels.append(label)
42
-
43
- formatter_section = config.get("pipeline", {}).get("result_formatter", {})
44
- if "abandon" in formatter_section:
45
- abandon = formatter_section["abandon"]
46
- if isinstance(abandon, list):
47
- formatter_section["abandon"] = [x for x in abandon if x not in to_include_as_text]
48
- # Include header/footer in visualization/output text category
49
- if "label_visualization_mapping" in formatter_section:
50
- text_vis = formatter_section["label_visualization_mapping"].get("text")
51
- if isinstance(text_vis, list):
52
- for label in to_include_as_text:
53
- if label not in text_vis:
54
- text_vis.append(label)
55
-
56
- with open(CONFIG_PATH, "w") as f:
57
- yaml.dump(config, f, default_flow_style=False, sort_keys=False)
58
 
59
  # ---------------------------------------------------------------------------
60
- # 2. result_formatter: stop stripping header/footer in postprocess
61
  # ---------------------------------------------------------------------------
62
- with open(FORMATTER_PATH, "r") as f:
63
- source = f.read()
64
- for label in ('"header"', "'header'", '"footer"', "'footer'", '"doc_header"', "'doc_header'", '"doc_footer"', "'doc_footer'"):
65
- source = re.sub(r",\s*" + re.escape(label), "", source)
66
- source = re.sub(re.escape(label) + r"\s*,", "", source)
67
- source = re.sub(re.escape(label), "", source)
68
- with open(FORMATTER_PATH, "w") as f:
69
- f.write(source)
 
 
 
70
 
71
 
72
  def get_header_footer_zones(regions, norm_height=1000):
@@ -134,7 +138,9 @@ def get_page_md_and_regions(page_result):
134
  regions = []
135
  if hasattr(page_result, "json_result"):
136
  jr = page_result.json_result
137
- if isinstance(jr, list) and len(jr) > 0:
 
 
138
  r = jr[0] if isinstance(jr[0], list) else jr
139
  if isinstance(r, list):
140
  regions = r
@@ -175,16 +181,18 @@ def run_ocr(uploaded_file):
175
  page_md, regions = get_page_md_and_regions(page_result)
176
  header_end_frac, footer_start_frac = get_header_footer_zones(regions, 1000)
177
 
 
178
  he = header_end_frac if header_end_frac is not None else DEFAULT_ZONE_FRAC
179
  fs = footer_start_frac if footer_start_frac is not None else (1.0 - DEFAULT_ZONE_FRAC)
180
- if he <= 0 and header_end_frac is not None:
181
- he = 0
182
- if fs >= 1.0 and footer_start_frac is not None:
183
- fs = 1.0
184
 
185
  parts = []
186
 
187
- if he > 0 and page_num < len(page_images):
 
188
  img_path = page_images[page_num]
189
  hdr = ""
190
  if is_pdf:
@@ -197,7 +205,8 @@ def run_ocr(uploaded_file):
197
  if page_md:
198
  parts.append(page_md)
199
 
200
- if fs < 1.0 and page_num < len(page_images):
 
201
  img_path = page_images[page_num]
202
  ftr = ""
203
  if is_pdf:
 
14
  CONFIG_PATH = os.path.join(GLMOCR_BASE, "config.yaml")
15
  FORMATTER_PATH = os.path.join(GLMOCR_BASE, "postprocess", "result_formatter.py")
16
 
17
+ # When MaaS is enabled, the cloud API ignores local layout config and does not return
18
+ # header/footer regions. We always run client-side OCR on top/bottom bands as fallback.
19
+ DEFAULT_ZONE_FRAC = float(os.environ.get("GLMOCR_DEFAULT_ZONE_FRAC", "0.12"))
20
 
21
  # ---------------------------------------------------------------------------
22
+ # 1. Config: set MaaS and optionally include headers/footers for non-MaaS
23
  # ---------------------------------------------------------------------------
24
+ try:
25
+ with open(CONFIG_PATH, "r") as f:
26
+ config = yaml.safe_load(f)
27
+ config["pipeline"]["maas"]["enabled"] = True
28
+ config["pipeline"]["maas"]["api_key"] = os.environ.get("GLMOCR_API_KEY", "4570c28bdea5493c9efae9dae68edc66.sGbA9DLlcX1GlvqV")
29
+
30
+ to_include_as_text = {"header", "footer"}
31
+ layout_section = config.get("pipeline", {}).get("layout", {})
32
+ if "label_task_mapping" in layout_section:
33
+ mapping = layout_section["label_task_mapping"]
34
+ abandon = mapping.get("abandon")
35
+ if isinstance(abandon, list):
36
+ mapping["abandon"] = [x for x in abandon if x not in to_include_as_text]
37
+ text_labels = mapping.get("text")
38
+ if isinstance(text_labels, list):
39
+ for label in to_include_as_text:
40
+ if label not in text_labels:
41
+ text_labels.append(label)
42
+
43
+ formatter_section = config.get("pipeline", {}).get("result_formatter", {})
44
+ if "abandon" in formatter_section:
45
+ abandon = formatter_section["abandon"]
46
+ if isinstance(abandon, list):
47
+ formatter_section["abandon"] = [x for x in abandon if x not in to_include_as_text]
48
+ if "label_visualization_mapping" in formatter_section:
49
+ text_vis = formatter_section["label_visualization_mapping"].get("text")
50
+ if isinstance(text_vis, list):
51
+ for label in to_include_as_text:
52
+ if label not in text_vis:
53
+ text_vis.append(label)
54
+
55
+ with open(CONFIG_PATH, "w") as f:
56
+ yaml.dump(config, f, default_flow_style=False, sort_keys=False)
57
+ except Exception:
58
+ pass # e.g. read-only package on Hugging Face; MaaS ignores layout anyway
 
59
 
60
  # ---------------------------------------------------------------------------
61
+ # 2. result_formatter: stop stripping header/footer (best-effort; may be read-only)
62
  # ---------------------------------------------------------------------------
63
+ try:
64
+ with open(FORMATTER_PATH, "r") as f:
65
+ source = f.read()
66
+ for label in ('"header"', "'header'", '"footer"', "'footer'", '"doc_header"', "'doc_header'", '"doc_footer"', "'doc_footer'"):
67
+ source = re.sub(r",\s*" + re.escape(label), "", source)
68
+ source = re.sub(re.escape(label) + r"\s*,", "", source)
69
+ source = re.sub(re.escape(label), "", source)
70
+ with open(FORMATTER_PATH, "w") as f:
71
+ f.write(source)
72
+ except Exception:
73
+ pass
74
 
75
 
76
  def get_header_footer_zones(regions, norm_height=1000):
 
138
  regions = []
139
  if hasattr(page_result, "json_result"):
140
  jr = page_result.json_result
141
+ if isinstance(jr, dict) and "regions" in jr:
142
+ regions = jr.get("regions") or []
143
+ elif isinstance(jr, list) and len(jr) > 0:
144
  r = jr[0] if isinstance(jr[0], list) else jr
145
  if isinstance(r, list):
146
  regions = r
 
181
  page_md, regions = get_page_md_and_regions(page_result)
182
  header_end_frac, footer_start_frac = get_header_footer_zones(regions, 1000)
183
 
184
+ # MaaS does not return header/footer regions; always use at least default bands
185
  he = header_end_frac if header_end_frac is not None else DEFAULT_ZONE_FRAC
186
  fs = footer_start_frac if footer_start_frac is not None else (1.0 - DEFAULT_ZONE_FRAC)
187
+ if he <= 0:
188
+ he = DEFAULT_ZONE_FRAC # ensure we always OCR a top band
189
+ if fs >= 1.0:
190
+ fs = 1.0 - DEFAULT_ZONE_FRAC # ensure we always OCR a bottom band
191
 
192
  parts = []
193
 
194
+ # Always run header band (top 8–12% of page)
195
+ if page_num < len(page_images):
196
  img_path = page_images[page_num]
197
  hdr = ""
198
  if is_pdf:
 
205
  if page_md:
206
  parts.append(page_md)
207
 
208
+ # Always run footer band (bottom 8–12% of page)
209
+ if page_num < len(page_images):
210
  img_path = page_images[page_num]
211
  ftr = ""
212
  if is_pdf: