from __future__ import annotations
import csv
import json
import importlib.util
from pathlib import Path
from ocrtech.cli import main
from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, write_manifest
from ocrtech.source_audit import audit_gorkhapatra_language_pages, audit_gorkhapatra_source, prepare_gorkhapatra_language_page_pack
def test_audit_gorkhapatra_source_parses_local_category_and_epaper_html(tmp_path: Path) -> None:
category = tmp_path / "category.html"
category.write_text(
"""
समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित संस्कृत भाषा
""",
encoding="utf-8",
)
epaper = tmp_path / "epaper.html"
epaper.write_text(
"""
२०८३ असार १ सोमवार
""",
encoding="utf-8",
)
audit = audit_gorkhapatra_source(
tmp_path / "out",
category_html_paths=[category],
epaper_html_paths=[epaper],
)
assert audit.article_count == 3
assert audit.epaper_count == 1
assert audit.language_counts == {"बान्तावा": 1, "मगर": 1, "संस्कृत": 1}
assert audit.articles[1].article_url == "https://gorkhapatraonline.com/news/207983"
assert audit.articles[2].article_url == "https://gorkhapatraonline.com/news/210293"
assert audit.epapers[0].direct_pdf_url == "https://epaper.gorkhapatraonline.com/uploads/file/2026/6/gorkhapatra/sample.pdf"
assert (tmp_path / "out" / "gorkhapatra-source-audit.json").exists()
assert (tmp_path / "out" / "gorkhapatra-source-audit.md").exists()
def test_cli_audit_gorkhapatra_source_downloads_local_article_image(tmp_path: Path, capsys) -> None:
image = tmp_path / "source.jpg"
image.write_bytes(b"not really a jpeg")
category = tmp_path / "category.html"
category.write_text(
f"""
""",
encoding="utf-8",
)
exit_code = main(
[
"audit-gorkhapatra-source",
"--no-default-urls",
"--category-html",
str(category),
"--download-assets",
"--max-article-images",
"1",
"--out",
str(tmp_path / "audit"),
]
)
captured = capsys.readouterr()
payload = json.loads((tmp_path / "audit" / "gorkhapatra-source-audit.json").read_text(encoding="utf-8"))
assert exit_code == 0
assert "articles=1 epapers=0 downloaded=1" in captured.out
assert payload["language_counts"] == {"लिम्बु": 1}
downloaded = payload["articles"][0]["downloaded_image"]
assert downloaded["sha256"]
assert Path(downloaded["path"]).read_bytes() == b"not really a jpeg"
def test_prepare_gorkhapatra_pack_writes_pending_reference_manifest_from_article_images(tmp_path: Path, capsys) -> None:
image = tmp_path / "source.jpg"
image.write_bytes(b"article image")
audit = tmp_path / "source-audit.json"
audit.write_text(
json.dumps(
{
"articles": [
{
"article_url": "https://gorkhapatraonline.com/news/limbu",
"image_url": "https://gorkhapatraonline.com/storage/limbu.jpg",
"title": "समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित लिम्बु भाषा",
"language": "लिम्बु",
"source_page": "fixture",
"downloaded_image": {
"url": "https://gorkhapatraonline.com/storage/limbu.jpg",
"path": str(image),
"sha256": "fixture",
"size_bytes": image.stat().st_size,
"kind": "image",
"warnings": [],
},
}
],
"epapers": [],
}
),
encoding="utf-8",
)
exit_code = main(
[
"prepare-gorkhapatra-pack",
str(audit),
"--out",
str(tmp_path / "pack"),
"--no-epaper-pages",
]
)
captured = capsys.readouterr()
entries = load_manifest(tmp_path / "pack" / "manifests" / "gorkhapatra-pack.jsonl")
reference = json.loads(Path(entries[0].metadata["reference_path"]).read_text(encoding="utf-8"))
assert exit_code == 0
assert "samples=1 article_images=1 epaper_pages=0" in captured.out
assert entries[0].metadata["language"] == "लिम्बु"
assert entries[0].metadata["reference_status"] == "pending_manual_label"
assert entries[0].metadata["claim_evidence_eligible"] is False
assert "language:लिम्बु" in entries[0].metadata["slices"]
assert Path(entries[0].image_path).read_bytes() == b"article image"
assert reference["metadata"]["reference_status"] == "pending_manual_label"
def test_prepare_gorkhapatra_pack_renders_epaper_pdf_pages_when_pymupdf_available(tmp_path: Path) -> None:
if importlib.util.find_spec("fitz") is None:
return
import fitz
pdf = tmp_path / "sample.pdf"
document = fitz.open()
page = document.new_page(width=120, height=120)
page.insert_text((20, 40), "Gorkhapatra")
document.save(pdf)
document.close()
audit = tmp_path / "source-audit.json"
audit.write_text(
json.dumps(
{
"articles": [],
"epapers": [
{
"viewer_url": "https://epaper.gorkhapatraonline.com/pdf?file=/uploads/file/sample.pdf",
"direct_pdf_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf",
"source_page": "fixture",
"date_label": "२०८३ असार १ सोमवार",
"downloaded_pdf": {
"url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf",
"path": str(pdf),
"sha256": "fixture",
"size_bytes": pdf.stat().st_size,
"kind": "pdf",
"warnings": [],
},
}
],
}
),
encoding="utf-8",
)
exit_code = main(
[
"prepare-gorkhapatra-pack",
str(audit),
"--out",
str(tmp_path / "pack"),
"--no-article-images",
"--max-epaper-pdfs",
"1",
"--max-pages-per-pdf",
"1",
]
)
entries = load_manifest(tmp_path / "pack" / "manifests" / "gorkhapatra-pack.jsonl")
assert exit_code == 0
assert len(entries) == 1
assert Path(entries[0].image_path).suffix == ".png"
assert Path(entries[0].image_path).exists()
assert entries[0].metadata["source_kind"] == "epaper_page"
assert entries[0].metadata["pdf_page_index"] == 0
def test_audit_gorkhapatra_language_pages_matches_publication_date_to_epaper(tmp_path: Path, capsys) -> None:
publication = tmp_path / "publication.html"
publication.write_text(
"""
- गोरखापत्र अनलाइन
- १ असार २०८३, सोमबार
""",
encoding="utf-8",
)
epaper = tmp_path / "epaper.html"
epaper.write_text(
"""
२०८३ असार १ सोमवार
""",
encoding="utf-8",
)
exit_code = main(
[
"audit-gorkhapatra-language-pages",
"--no-default-urls",
"--publication-html",
str(publication),
"--epaper-html",
str(epaper),
"--out",
str(tmp_path / "out"),
]
)
captured = capsys.readouterr()
payload = json.loads((tmp_path / "out" / "gorkhapatra-language-pages.json").read_text(encoding="utf-8"))
alignment = payload["alignments"][0]
assert exit_code == 0
assert "publication_items=1 aligned=1 language_page_hits=0" in captured.out
assert payload["status_counts"] == {"matched_epaper_not_downloaded": 1}
assert alignment["publication_item"]["date_key"] == "2083-03-01"
assert alignment["publication_item"]["language"] == "मगर"
assert alignment["epaper"]["direct_pdf_url"] == "https://epaper.gorkhapatraonline.com/uploads/file/2026/6/gorkhapatra/sample.pdf"
assert alignment["claim_evidence_eligible"] is False
def test_audit_gorkhapatra_language_pages_finds_marker_in_downloaded_pdf(tmp_path: Path) -> None:
if importlib.util.find_spec("fitz") is None:
return
import fitz
pdf = tmp_path / "language-page.pdf"
document = fitz.open()
page = document.new_page(width=300, height=300)
page.insert_text((40, 80), "bhasa pristha")
page.insert_text((40, 120), "Sample language page body")
document.save(pdf)
document.close()
publication = tmp_path / "publication.html"
publication.write_text(
"""
""",
encoding="utf-8",
)
epaper = tmp_path / "epaper.html"
epaper.write_text(
f"""
२०८३ असार १ सोमवार
""",
encoding="utf-8",
)
audit = audit_gorkhapatra_language_pages(
tmp_path / "out",
publication_html_paths=[publication],
epaper_html_paths=[epaper],
download_pdfs=True,
render_pages=True,
)
assert audit.language_page_hit_count == 1
assert audit.alignments[0].alignment_status == "language_page_found"
assert audit.alignments[0].page_hits[0].page_index == 0
assert audit.alignments[0].page_hits[0].rendered_page_path is not None
assert Path(audit.alignments[0].page_hits[0].rendered_page_path).exists()
def test_prepare_gorkhapatra_language_page_pack_writes_pending_manifest(tmp_path: Path, capsys) -> None:
rendered = tmp_path / "page.png"
rendered.write_bytes(b"rendered page")
audit = tmp_path / "language-pages.json"
audit.write_text(
json.dumps(
{
"alignments": [
{
"publication_item": {
"article_url": "https://gorkhapatraonline.com/news/210290",
"image_url": "https://gorkhapatraonline.com/storage/media/365132/6.jpg",
"title": "समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित मगर भाषा",
"language": "मगर",
"date_label": "१ असार २०८३, सोमबार",
"date_key": "2083-03-01",
"source_page": "fixture",
},
"epaper": {
"viewer_url": "https://epaper.gorkhapatraonline.com/pdf?file=/uploads/file/sample.pdf",
"direct_pdf_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf",
"source_page": "fixture",
"date_label": "२०८३ असार १ सोमवार",
"thumbnail_url": None,
"downloaded_pdf": None,
},
"pdf_path": str(tmp_path / "sample.pdf"),
"pdf_sha256": "pdfhash",
"page_hits": [
{
"page_index": 5,
"marker": "efiff k[i7",
"text_chars": 120,
"text_preview": "efiff k[i7",
"rendered_page_path": str(rendered),
}
],
"alignment_status": "language_page_found",
"claim_evidence_eligible": False,
"warnings": [],
}
]
}
),
encoding="utf-8",
)
exit_code = main(
[
"prepare-gorkhapatra-language-page-pack",
str(audit),
"--out",
str(tmp_path / "pack"),
]
)
captured = capsys.readouterr()
entries = load_manifest(tmp_path / "pack" / "manifests" / "gorkhapatra-language-page-pack.jsonl")
reference = json.loads(Path(entries[0].metadata["reference_path"]).read_text(encoding="utf-8"))
assert exit_code == 0
assert "samples=1 copied_pages=1 warnings=0" in captured.out
assert len(entries) == 1
assert Path(entries[0].image_path).read_bytes() == b"rendered page"
assert entries[0].metadata["candidate_language"] == "मगर"
assert entries[0].metadata["pdf_page_number"] == 6
assert entries[0].metadata["page_disambiguation_status"] == "candidate_requires_manual_review"
assert entries[0].metadata["reference_status"] == "pending_manual_label"
assert entries[0].metadata["claim_evidence_eligible"] is False
assert "language_page_candidate" in entries[0].metadata["slices"]
assert reference["metadata"]["source_kind"] == "gorkhapatra_language_page_candidate"
def test_prepare_gorkhapatra_language_page_pack_records_missing_rendered_pages(tmp_path: Path) -> None:
audit = tmp_path / "language-pages.json"
audit.write_text(
json.dumps(
{
"alignments": [
{
"publication_item": {"language": "मगर", "date_key": "2083-03-01"},
"page_hits": [{"page_index": 5, "marker": "efiff k[i7"}],
}
]
}
),
encoding="utf-8",
)
summary = prepare_gorkhapatra_language_page_pack(audit, tmp_path / "pack")
assert summary.sample_count == 0
assert summary.warnings == ["alignment 1 page hit 1 has no rendered_page_path"]
def test_prepare_gorkhapatra_language_page_review_writes_manual_review_artifacts(tmp_path: Path, capsys) -> None:
image_one = tmp_path / "page-one.png"
image_two = tmp_path / "page-two.png"
image_one.write_bytes(b"candidate one")
image_two.write_bytes(b"candidate two")
ref_one = tmp_path / "page-one.ref.json"
ref_two = tmp_path / "page-two.ref.json"
ref_one.write_text("{}", encoding="utf-8")
ref_two.write_text("{}", encoding="utf-8")
manifest = tmp_path / "manifest.jsonl"
write_manifest(
[
ManifestEntry(
sample_id="gorkhapatra-language-page-0001-01",
dataset="fixture",
split="eval",
image_path=str(image_one),
text="",
sha256=sha256_file(image_one),
metadata={
"source_kind": "language_page_candidate",
"candidate_language": "संस्कृत",
"date_label": "१ असार २०८३, सोमबार",
"date_key": "2083-03-01",
"pdf_page_index": 5,
"pdf_page_number": 6,
"reference_path": str(ref_one),
"article_url": "https://gorkhapatraonline.com/news/1",
"source_image_url": "https://gorkhapatraonline.com/storage/1.jpg",
"source_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf",
"pdf_path": str(tmp_path / "sample.pdf"),
"pdf_sha256": "pdfhash",
"language_page_marker": "efiff k[i7",
"alignment_status": "language_page_found",
"page_disambiguation_status": "candidate_requires_manual_review",
"reference_status": "pending_manual_label",
"claim_evidence_eligible": False,
},
),
ManifestEntry(
sample_id="gorkhapatra-language-page-0002-01",
dataset="fixture",
split="eval",
image_path=str(image_two),
text="",
sha256=sha256_file(image_two),
metadata={
"source_kind": "language_page_candidate",
"candidate_language": "मगर",
"date_key": "2083-03-01",
"pdf_page_index": 5,
"pdf_page_number": 6,
"reference_path": str(ref_two),
"source_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf",
"language_page_marker": "efiff k[i7",
"alignment_status": "language_page_found",
"page_disambiguation_status": "candidate_requires_manual_review",
"reference_status": "pending_manual_label",
"claim_evidence_eligible": False,
},
),
],
manifest,
)
exit_code = main(
[
"prepare-gorkhapatra-language-page-review",
str(manifest),
"--out",
str(tmp_path / "review"),
]
)
captured = capsys.readouterr()
payload = json.loads((tmp_path / "review" / "gorkhapatra-language-page-review.json").read_text(encoding="utf-8"))
with (tmp_path / "review" / "gorkhapatra-language-page-review.csv").open("r", encoding="utf-8", newline="") as handle:
csv_rows = list(csv.DictReader(handle))
markdown = (tmp_path / "review" / "gorkhapatra-language-page-review.md").read_text(encoding="utf-8")
assert exit_code == 0
assert "samples=2 pending_review=2 claim_eligible=0 warnings=0" in captured.out
assert payload["summary"]["sample_count"] == 2
assert payload["summary"]["page_candidate_counts"] == {"2083-03-01:page-6": 2}
assert payload["rows"][0]["review_decision"] == ""
assert payload["rows"][0]["claim_evidence_eligible"] is False
assert csv_rows[1]["candidate_language"] == "मगर"
assert csv_rows[1]["reviewer"] == ""
assert "Rows remain non-claim-ready" in markdown