Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download tests/test_source_audit.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 22.3 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_source_audit.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/tests/test_source_audit.py
-
curl -L -o test_source_audit.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/tests/test_source_audit.py
22.3 kB
| from __future__ import annotations | |
| import csv | |
| import json | |
| import importlib.util | |
| from pathlib import Path | |
| from ocrtech.cli import main | |
| from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, write_manifest | |
| from ocrtech.source_audit import audit_gorkhapatra_language_pages, audit_gorkhapatra_source, prepare_gorkhapatra_language_page_pack | |
| def test_audit_gorkhapatra_source_parses_local_category_and_epaper_html(tmp_path: Path) -> None: | |
| category = tmp_path / "category.html" | |
| category.write_text( | |
| """ | |
| <html><body> | |
| <a href="https://gorkhapatraonline.com/news/210290"> | |
| <img alt="समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित मगर भाषा" | |
| src="https://gorkhapatraonline.com/storage/media/365132/conversions/6-original-thumb.jpg"> | |
| </a> | |
| <a href="/news/207983"> | |
| <img alt="समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित बान्तावा भाषा" | |
| src="/storage/media/fixture/bantawa.jpg"> | |
| </a> | |
| <img alt="समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित संस्कृत भाषा" | |
| src="/storage/media/fixture/sanskrit.jpg"> | |
| <a href="/news/210293">समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित संस्कृत भाषा</a> | |
| </body></html> | |
| """, | |
| encoding="utf-8", | |
| ) | |
| epaper = tmp_path / "epaper.html" | |
| epaper.write_text( | |
| """ | |
| <html><body> | |
| <a href="https://epaper.gorkhapatraonline.com/pdf?file=/uploads/file/2026/6/gorkhapatra/sample.pdf"> | |
| <img src="/uploads/image/2026/6/gorkhapatra/thumb/sample.jpg"> | |
| <span class="date">२०८३ असार १ सोमवार</span> | |
| </a> | |
| </body></html> | |
| """, | |
| encoding="utf-8", | |
| ) | |
| audit = audit_gorkhapatra_source( | |
| tmp_path / "out", | |
| category_html_paths=[category], | |
| epaper_html_paths=[epaper], | |
| ) | |
| assert audit.article_count == 3 | |
| assert audit.epaper_count == 1 | |
| assert audit.language_counts == {"बान्तावा": 1, "मगर": 1, "संस्कृत": 1} | |
| assert audit.articles[1].article_url == "https://gorkhapatraonline.com/news/207983" | |
| assert audit.articles[2].article_url == "https://gorkhapatraonline.com/news/210293" | |
| assert audit.epapers[0].direct_pdf_url == "https://epaper.gorkhapatraonline.com/uploads/file/2026/6/gorkhapatra/sample.pdf" | |
| assert (tmp_path / "out" / "gorkhapatra-source-audit.json").exists() | |
| assert (tmp_path / "out" / "gorkhapatra-source-audit.md").exists() | |
| def test_cli_audit_gorkhapatra_source_downloads_local_article_image(tmp_path: Path, capsys) -> None: | |
| image = tmp_path / "source.jpg" | |
| image.write_bytes(b"not really a jpeg") | |
| category = tmp_path / "category.html" | |
| category.write_text( | |
| f""" | |
| <html><body> | |
| <a href="https://gorkhapatraonline.com/news/1"> | |
| <img alt="समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित लिम्बु भाषा" | |
| src="{image.as_uri()}"> | |
| </a> | |
| </body></html> | |
| """, | |
| encoding="utf-8", | |
| ) | |
| exit_code = main( | |
| [ | |
| "audit-gorkhapatra-source", | |
| "--no-default-urls", | |
| "--category-html", | |
| str(category), | |
| "--download-assets", | |
| "--max-article-images", | |
| "1", | |
| "--out", | |
| str(tmp_path / "audit"), | |
| ] | |
| ) | |
| captured = capsys.readouterr() | |
| payload = json.loads((tmp_path / "audit" / "gorkhapatra-source-audit.json").read_text(encoding="utf-8")) | |
| assert exit_code == 0 | |
| assert "articles=1 epapers=0 downloaded=1" in captured.out | |
| assert payload["language_counts"] == {"लिम्बु": 1} | |
| downloaded = payload["articles"][0]["downloaded_image"] | |
| assert downloaded["sha256"] | |
| assert Path(downloaded["path"]).read_bytes() == b"not really a jpeg" | |
| def test_prepare_gorkhapatra_pack_writes_pending_reference_manifest_from_article_images(tmp_path: Path, capsys) -> None: | |
| image = tmp_path / "source.jpg" | |
| image.write_bytes(b"article image") | |
| audit = tmp_path / "source-audit.json" | |
| audit.write_text( | |
| json.dumps( | |
| { | |
| "articles": [ | |
| { | |
| "article_url": "https://gorkhapatraonline.com/news/limbu", | |
| "image_url": "https://gorkhapatraonline.com/storage/limbu.jpg", | |
| "title": "समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित लिम्बु भाषा", | |
| "language": "लिम्बु", | |
| "source_page": "fixture", | |
| "downloaded_image": { | |
| "url": "https://gorkhapatraonline.com/storage/limbu.jpg", | |
| "path": str(image), | |
| "sha256": "fixture", | |
| "size_bytes": image.stat().st_size, | |
| "kind": "image", | |
| "warnings": [], | |
| }, | |
| } | |
| ], | |
| "epapers": [], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| exit_code = main( | |
| [ | |
| "prepare-gorkhapatra-pack", | |
| str(audit), | |
| "--out", | |
| str(tmp_path / "pack"), | |
| "--no-epaper-pages", | |
| ] | |
| ) | |
| captured = capsys.readouterr() | |
| entries = load_manifest(tmp_path / "pack" / "manifests" / "gorkhapatra-pack.jsonl") | |
| reference = json.loads(Path(entries[0].metadata["reference_path"]).read_text(encoding="utf-8")) | |
| assert exit_code == 0 | |
| assert "samples=1 article_images=1 epaper_pages=0" in captured.out | |
| assert entries[0].metadata["language"] == "लिम्बु" | |
| assert entries[0].metadata["reference_status"] == "pending_manual_label" | |
| assert entries[0].metadata["claim_evidence_eligible"] is False | |
| assert "language:लिम्बु" in entries[0].metadata["slices"] | |
| assert Path(entries[0].image_path).read_bytes() == b"article image" | |
| assert reference["metadata"]["reference_status"] == "pending_manual_label" | |
| def test_prepare_gorkhapatra_pack_renders_epaper_pdf_pages_when_pymupdf_available(tmp_path: Path) -> None: | |
| if importlib.util.find_spec("fitz") is None: | |
| return | |
| import fitz | |
| pdf = tmp_path / "sample.pdf" | |
| document = fitz.open() | |
| page = document.new_page(width=120, height=120) | |
| page.insert_text((20, 40), "Gorkhapatra") | |
| document.save(pdf) | |
| document.close() | |
| audit = tmp_path / "source-audit.json" | |
| audit.write_text( | |
| json.dumps( | |
| { | |
| "articles": [], | |
| "epapers": [ | |
| { | |
| "viewer_url": "https://epaper.gorkhapatraonline.com/pdf?file=/uploads/file/sample.pdf", | |
| "direct_pdf_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf", | |
| "source_page": "fixture", | |
| "date_label": "२०८३ असार १ सोमवार", | |
| "downloaded_pdf": { | |
| "url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf", | |
| "path": str(pdf), | |
| "sha256": "fixture", | |
| "size_bytes": pdf.stat().st_size, | |
| "kind": "pdf", | |
| "warnings": [], | |
| }, | |
| } | |
| ], | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| exit_code = main( | |
| [ | |
| "prepare-gorkhapatra-pack", | |
| str(audit), | |
| "--out", | |
| str(tmp_path / "pack"), | |
| "--no-article-images", | |
| "--max-epaper-pdfs", | |
| "1", | |
| "--max-pages-per-pdf", | |
| "1", | |
| ] | |
| ) | |
| entries = load_manifest(tmp_path / "pack" / "manifests" / "gorkhapatra-pack.jsonl") | |
| assert exit_code == 0 | |
| assert len(entries) == 1 | |
| assert Path(entries[0].image_path).suffix == ".png" | |
| assert Path(entries[0].image_path).exists() | |
| assert entries[0].metadata["source_kind"] == "epaper_page" | |
| assert entries[0].metadata["pdf_page_index"] == 0 | |
| def test_audit_gorkhapatra_language_pages_matches_publication_date_to_epaper(tmp_path: Path, capsys) -> None: | |
| publication = tmp_path / "publication.html" | |
| publication.write_text( | |
| """ | |
| <html><body> | |
| <div class="blog-box-layout1 publication"> | |
| <div class="item-img"> | |
| <a href="https://gorkhapatraonline.com/news/210290"> | |
| <img alt="समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित मगर भाषा" | |
| src="https://gorkhapatraonline.com/storage/media/365132/conversions/6-original-thumb.jpg"> | |
| </a> | |
| </div> | |
| <div class="item-content"> | |
| <ul class="entry-meta"> | |
| <li><i class="fas fa-user"></i>गोरखापत्र अनलाइन</li> | |
| <li><i class="fas fa-calendar-alt"></i>१ असार २०८३, सोमबार</li> | |
| </ul> | |
| <h3 class="item-title"> | |
| <a href="https://gorkhapatraonline.com/news/210290">समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित मगर भाषा</a> | |
| </h3> | |
| </div> | |
| </div> | |
| </body></html> | |
| """, | |
| encoding="utf-8", | |
| ) | |
| epaper = tmp_path / "epaper.html" | |
| epaper.write_text( | |
| """ | |
| <html><body> | |
| <a href="https://epaper.gorkhapatraonline.com/pdf?file=/uploads/file/2026/6/gorkhapatra/sample.pdf"> | |
| <span class="date">२०८३ असार १ सोमवार</span> | |
| </a> | |
| </body></html> | |
| """, | |
| encoding="utf-8", | |
| ) | |
| exit_code = main( | |
| [ | |
| "audit-gorkhapatra-language-pages", | |
| "--no-default-urls", | |
| "--publication-html", | |
| str(publication), | |
| "--epaper-html", | |
| str(epaper), | |
| "--out", | |
| str(tmp_path / "out"), | |
| ] | |
| ) | |
| captured = capsys.readouterr() | |
| payload = json.loads((tmp_path / "out" / "gorkhapatra-language-pages.json").read_text(encoding="utf-8")) | |
| alignment = payload["alignments"][0] | |
| assert exit_code == 0 | |
| assert "publication_items=1 aligned=1 language_page_hits=0" in captured.out | |
| assert payload["status_counts"] == {"matched_epaper_not_downloaded": 1} | |
| assert alignment["publication_item"]["date_key"] == "2083-03-01" | |
| assert alignment["publication_item"]["language"] == "मगर" | |
| assert alignment["epaper"]["direct_pdf_url"] == "https://epaper.gorkhapatraonline.com/uploads/file/2026/6/gorkhapatra/sample.pdf" | |
| assert alignment["claim_evidence_eligible"] is False | |
| def test_audit_gorkhapatra_language_pages_finds_marker_in_downloaded_pdf(tmp_path: Path) -> None: | |
| if importlib.util.find_spec("fitz") is None: | |
| return | |
| import fitz | |
| pdf = tmp_path / "language-page.pdf" | |
| document = fitz.open() | |
| page = document.new_page(width=300, height=300) | |
| page.insert_text((40, 80), "bhasa pristha") | |
| page.insert_text((40, 120), "Sample language page body") | |
| document.save(pdf) | |
| document.close() | |
| publication = tmp_path / "publication.html" | |
| publication.write_text( | |
| """ | |
| <html><body> | |
| <a href="https://gorkhapatraonline.com/news/1"> | |
| <img alt="समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित लिम्बु भाषा" src="/limbu.jpg"> | |
| </a> | |
| <ul><li><i class="fas fa-calendar-alt"></i>१ असार २०८३, सोमबार</li></ul> | |
| </body></html> | |
| """, | |
| encoding="utf-8", | |
| ) | |
| epaper = tmp_path / "epaper.html" | |
| epaper.write_text( | |
| f""" | |
| <html><body> | |
| <a href="https://epaper.gorkhapatraonline.com/pdf?file={pdf.as_uri()}"> | |
| <span class="date">२०८३ असार १ सोमवार</span> | |
| </a> | |
| </body></html> | |
| """, | |
| encoding="utf-8", | |
| ) | |
| audit = audit_gorkhapatra_language_pages( | |
| tmp_path / "out", | |
| publication_html_paths=[publication], | |
| epaper_html_paths=[epaper], | |
| download_pdfs=True, | |
| render_pages=True, | |
| ) | |
| assert audit.language_page_hit_count == 1 | |
| assert audit.alignments[0].alignment_status == "language_page_found" | |
| assert audit.alignments[0].page_hits[0].page_index == 0 | |
| assert audit.alignments[0].page_hits[0].rendered_page_path is not None | |
| assert Path(audit.alignments[0].page_hits[0].rendered_page_path).exists() | |
| def test_prepare_gorkhapatra_language_page_pack_writes_pending_manifest(tmp_path: Path, capsys) -> None: | |
| rendered = tmp_path / "page.png" | |
| rendered.write_bytes(b"rendered page") | |
| audit = tmp_path / "language-pages.json" | |
| audit.write_text( | |
| json.dumps( | |
| { | |
| "alignments": [ | |
| { | |
| "publication_item": { | |
| "article_url": "https://gorkhapatraonline.com/news/210290", | |
| "image_url": "https://gorkhapatraonline.com/storage/media/365132/6.jpg", | |
| "title": "समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित मगर भाषा", | |
| "language": "मगर", | |
| "date_label": "१ असार २०८३, सोमबार", | |
| "date_key": "2083-03-01", | |
| "source_page": "fixture", | |
| }, | |
| "epaper": { | |
| "viewer_url": "https://epaper.gorkhapatraonline.com/pdf?file=/uploads/file/sample.pdf", | |
| "direct_pdf_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf", | |
| "source_page": "fixture", | |
| "date_label": "२०८३ असार १ सोमवार", | |
| "thumbnail_url": None, | |
| "downloaded_pdf": None, | |
| }, | |
| "pdf_path": str(tmp_path / "sample.pdf"), | |
| "pdf_sha256": "pdfhash", | |
| "page_hits": [ | |
| { | |
| "page_index": 5, | |
| "marker": "efiff k[i7", | |
| "text_chars": 120, | |
| "text_preview": "efiff k[i7", | |
| "rendered_page_path": str(rendered), | |
| } | |
| ], | |
| "alignment_status": "language_page_found", | |
| "claim_evidence_eligible": False, | |
| "warnings": [], | |
| } | |
| ] | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| exit_code = main( | |
| [ | |
| "prepare-gorkhapatra-language-page-pack", | |
| str(audit), | |
| "--out", | |
| str(tmp_path / "pack"), | |
| ] | |
| ) | |
| captured = capsys.readouterr() | |
| entries = load_manifest(tmp_path / "pack" / "manifests" / "gorkhapatra-language-page-pack.jsonl") | |
| reference = json.loads(Path(entries[0].metadata["reference_path"]).read_text(encoding="utf-8")) | |
| assert exit_code == 0 | |
| assert "samples=1 copied_pages=1 warnings=0" in captured.out | |
| assert len(entries) == 1 | |
| assert Path(entries[0].image_path).read_bytes() == b"rendered page" | |
| assert entries[0].metadata["candidate_language"] == "मगर" | |
| assert entries[0].metadata["pdf_page_number"] == 6 | |
| assert entries[0].metadata["page_disambiguation_status"] == "candidate_requires_manual_review" | |
| assert entries[0].metadata["reference_status"] == "pending_manual_label" | |
| assert entries[0].metadata["claim_evidence_eligible"] is False | |
| assert "language_page_candidate" in entries[0].metadata["slices"] | |
| assert reference["metadata"]["source_kind"] == "gorkhapatra_language_page_candidate" | |
| def test_prepare_gorkhapatra_language_page_pack_records_missing_rendered_pages(tmp_path: Path) -> None: | |
| audit = tmp_path / "language-pages.json" | |
| audit.write_text( | |
| json.dumps( | |
| { | |
| "alignments": [ | |
| { | |
| "publication_item": {"language": "मगर", "date_key": "2083-03-01"}, | |
| "page_hits": [{"page_index": 5, "marker": "efiff k[i7"}], | |
| } | |
| ] | |
| } | |
| ), | |
| encoding="utf-8", | |
| ) | |
| summary = prepare_gorkhapatra_language_page_pack(audit, tmp_path / "pack") | |
| assert summary.sample_count == 0 | |
| assert summary.warnings == ["alignment 1 page hit 1 has no rendered_page_path"] | |
| def test_prepare_gorkhapatra_language_page_review_writes_manual_review_artifacts(tmp_path: Path, capsys) -> None: | |
| image_one = tmp_path / "page-one.png" | |
| image_two = tmp_path / "page-two.png" | |
| image_one.write_bytes(b"candidate one") | |
| image_two.write_bytes(b"candidate two") | |
| ref_one = tmp_path / "page-one.ref.json" | |
| ref_two = tmp_path / "page-two.ref.json" | |
| ref_one.write_text("{}", encoding="utf-8") | |
| ref_two.write_text("{}", encoding="utf-8") | |
| manifest = tmp_path / "manifest.jsonl" | |
| write_manifest( | |
| [ | |
| ManifestEntry( | |
| sample_id="gorkhapatra-language-page-0001-01", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(image_one), | |
| text="", | |
| sha256=sha256_file(image_one), | |
| metadata={ | |
| "source_kind": "language_page_candidate", | |
| "candidate_language": "संस्कृत", | |
| "date_label": "१ असार २०८३, सोमबार", | |
| "date_key": "2083-03-01", | |
| "pdf_page_index": 5, | |
| "pdf_page_number": 6, | |
| "reference_path": str(ref_one), | |
| "article_url": "https://gorkhapatraonline.com/news/1", | |
| "source_image_url": "https://gorkhapatraonline.com/storage/1.jpg", | |
| "source_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf", | |
| "pdf_path": str(tmp_path / "sample.pdf"), | |
| "pdf_sha256": "pdfhash", | |
| "language_page_marker": "efiff k[i7", | |
| "alignment_status": "language_page_found", | |
| "page_disambiguation_status": "candidate_requires_manual_review", | |
| "reference_status": "pending_manual_label", | |
| "claim_evidence_eligible": False, | |
| }, | |
| ), | |
| ManifestEntry( | |
| sample_id="gorkhapatra-language-page-0002-01", | |
| dataset="fixture", | |
| split="eval", | |
| image_path=str(image_two), | |
| text="", | |
| sha256=sha256_file(image_two), | |
| metadata={ | |
| "source_kind": "language_page_candidate", | |
| "candidate_language": "मगर", | |
| "date_key": "2083-03-01", | |
| "pdf_page_index": 5, | |
| "pdf_page_number": 6, | |
| "reference_path": str(ref_two), | |
| "source_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf", | |
| "language_page_marker": "efiff k[i7", | |
| "alignment_status": "language_page_found", | |
| "page_disambiguation_status": "candidate_requires_manual_review", | |
| "reference_status": "pending_manual_label", | |
| "claim_evidence_eligible": False, | |
| }, | |
| ), | |
| ], | |
| manifest, | |
| ) | |
| exit_code = main( | |
| [ | |
| "prepare-gorkhapatra-language-page-review", | |
| str(manifest), | |
| "--out", | |
| str(tmp_path / "review"), | |
| ] | |
| ) | |
| captured = capsys.readouterr() | |
| payload = json.loads((tmp_path / "review" / "gorkhapatra-language-page-review.json").read_text(encoding="utf-8")) | |
| with (tmp_path / "review" / "gorkhapatra-language-page-review.csv").open("r", encoding="utf-8", newline="") as handle: | |
| csv_rows = list(csv.DictReader(handle)) | |
| markdown = (tmp_path / "review" / "gorkhapatra-language-page-review.md").read_text(encoding="utf-8") | |
| assert exit_code == 0 | |
| assert "samples=2 pending_review=2 claim_eligible=0 warnings=0" in captured.out | |
| assert payload["summary"]["sample_count"] == 2 | |
| assert payload["summary"]["page_candidate_counts"] == {"2083-03-01:page-6": 2} | |
| assert payload["rows"][0]["review_decision"] == "" | |
| assert payload["rows"][0]["claim_evidence_eligible"] is False | |
| assert csv_rows[1]["candidate_language"] == "मगर" | |
| assert csv_rows[1]["reviewer"] == "" | |
| assert "Rows remain non-claim-ready" in markdown | |