from __future__ import annotations import csv import json import importlib.util from pathlib import Path from ocrtech.cli import main from ocrtech.manifest import ManifestEntry, load_manifest, sha256_file, write_manifest from ocrtech.source_audit import audit_gorkhapatra_language_pages, audit_gorkhapatra_source, prepare_gorkhapatra_language_page_pack def test_audit_gorkhapatra_source_parses_local_category_and_epaper_html(tmp_path: Path) -> None: category = tmp_path / "category.html" category.write_text( """ समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित मगर  भाषा समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित बान्तावा  भाषा समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित संस्कृत  भाषा समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित संस्कृत भाषा """, encoding="utf-8", ) epaper = tmp_path / "epaper.html" epaper.write_text( """ २०८३ असार १ सोमवार """, encoding="utf-8", ) audit = audit_gorkhapatra_source( tmp_path / "out", category_html_paths=[category], epaper_html_paths=[epaper], ) assert audit.article_count == 3 assert audit.epaper_count == 1 assert audit.language_counts == {"बान्तावा": 1, "मगर": 1, "संस्कृत": 1} assert audit.articles[1].article_url == "https://gorkhapatraonline.com/news/207983" assert audit.articles[2].article_url == "https://gorkhapatraonline.com/news/210293" assert audit.epapers[0].direct_pdf_url == "https://epaper.gorkhapatraonline.com/uploads/file/2026/6/gorkhapatra/sample.pdf" assert (tmp_path / "out" / "gorkhapatra-source-audit.json").exists() assert (tmp_path / "out" / "gorkhapatra-source-audit.md").exists() def test_cli_audit_gorkhapatra_source_downloads_local_article_image(tmp_path: Path, capsys) -> None: image = tmp_path / "source.jpg" image.write_bytes(b"not really a jpeg") category = tmp_path / "category.html" category.write_text( f""" समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित लिम्बु भाषा """, encoding="utf-8", ) exit_code = main( [ "audit-gorkhapatra-source", "--no-default-urls", "--category-html", str(category), "--download-assets", "--max-article-images", "1", "--out", str(tmp_path / "audit"), ] ) captured = capsys.readouterr() payload = json.loads((tmp_path / "audit" / "gorkhapatra-source-audit.json").read_text(encoding="utf-8")) assert exit_code == 0 assert "articles=1 epapers=0 downloaded=1" in captured.out assert payload["language_counts"] == {"लिम्बु": 1} downloaded = payload["articles"][0]["downloaded_image"] assert downloaded["sha256"] assert Path(downloaded["path"]).read_bytes() == b"not really a jpeg" def test_prepare_gorkhapatra_pack_writes_pending_reference_manifest_from_article_images(tmp_path: Path, capsys) -> None: image = tmp_path / "source.jpg" image.write_bytes(b"article image") audit = tmp_path / "source-audit.json" audit.write_text( json.dumps( { "articles": [ { "article_url": "https://gorkhapatraonline.com/news/limbu", "image_url": "https://gorkhapatraonline.com/storage/limbu.jpg", "title": "समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित लिम्बु भाषा", "language": "लिम्बु", "source_page": "fixture", "downloaded_image": { "url": "https://gorkhapatraonline.com/storage/limbu.jpg", "path": str(image), "sha256": "fixture", "size_bytes": image.stat().st_size, "kind": "image", "warnings": [], }, } ], "epapers": [], } ), encoding="utf-8", ) exit_code = main( [ "prepare-gorkhapatra-pack", str(audit), "--out", str(tmp_path / "pack"), "--no-epaper-pages", ] ) captured = capsys.readouterr() entries = load_manifest(tmp_path / "pack" / "manifests" / "gorkhapatra-pack.jsonl") reference = json.loads(Path(entries[0].metadata["reference_path"]).read_text(encoding="utf-8")) assert exit_code == 0 assert "samples=1 article_images=1 epaper_pages=0" in captured.out assert entries[0].metadata["language"] == "लिम्बु" assert entries[0].metadata["reference_status"] == "pending_manual_label" assert entries[0].metadata["claim_evidence_eligible"] is False assert "language:लिम्बु" in entries[0].metadata["slices"] assert Path(entries[0].image_path).read_bytes() == b"article image" assert reference["metadata"]["reference_status"] == "pending_manual_label" def test_prepare_gorkhapatra_pack_renders_epaper_pdf_pages_when_pymupdf_available(tmp_path: Path) -> None: if importlib.util.find_spec("fitz") is None: return import fitz pdf = tmp_path / "sample.pdf" document = fitz.open() page = document.new_page(width=120, height=120) page.insert_text((20, 40), "Gorkhapatra") document.save(pdf) document.close() audit = tmp_path / "source-audit.json" audit.write_text( json.dumps( { "articles": [], "epapers": [ { "viewer_url": "https://epaper.gorkhapatraonline.com/pdf?file=/uploads/file/sample.pdf", "direct_pdf_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf", "source_page": "fixture", "date_label": "२०८३ असार १ सोमवार", "downloaded_pdf": { "url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf", "path": str(pdf), "sha256": "fixture", "size_bytes": pdf.stat().st_size, "kind": "pdf", "warnings": [], }, } ], } ), encoding="utf-8", ) exit_code = main( [ "prepare-gorkhapatra-pack", str(audit), "--out", str(tmp_path / "pack"), "--no-article-images", "--max-epaper-pdfs", "1", "--max-pages-per-pdf", "1", ] ) entries = load_manifest(tmp_path / "pack" / "manifests" / "gorkhapatra-pack.jsonl") assert exit_code == 0 assert len(entries) == 1 assert Path(entries[0].image_path).suffix == ".png" assert Path(entries[0].image_path).exists() assert entries[0].metadata["source_kind"] == "epaper_page" assert entries[0].metadata["pdf_page_index"] == 0 def test_audit_gorkhapatra_language_pages_matches_publication_date_to_epaper(tmp_path: Path, capsys) -> None: publication = tmp_path / "publication.html" publication.write_text( """
समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित मगर  भाषा

समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित मगर भाषा

""", encoding="utf-8", ) epaper = tmp_path / "epaper.html" epaper.write_text( """ २०८३ असार १ सोमवार """, encoding="utf-8", ) exit_code = main( [ "audit-gorkhapatra-language-pages", "--no-default-urls", "--publication-html", str(publication), "--epaper-html", str(epaper), "--out", str(tmp_path / "out"), ] ) captured = capsys.readouterr() payload = json.loads((tmp_path / "out" / "gorkhapatra-language-pages.json").read_text(encoding="utf-8")) alignment = payload["alignments"][0] assert exit_code == 0 assert "publication_items=1 aligned=1 language_page_hits=0" in captured.out assert payload["status_counts"] == {"matched_epaper_not_downloaded": 1} assert alignment["publication_item"]["date_key"] == "2083-03-01" assert alignment["publication_item"]["language"] == "मगर" assert alignment["epaper"]["direct_pdf_url"] == "https://epaper.gorkhapatraonline.com/uploads/file/2026/6/gorkhapatra/sample.pdf" assert alignment["claim_evidence_eligible"] is False def test_audit_gorkhapatra_language_pages_finds_marker_in_downloaded_pdf(tmp_path: Path) -> None: if importlib.util.find_spec("fitz") is None: return import fitz pdf = tmp_path / "language-page.pdf" document = fitz.open() page = document.new_page(width=300, height=300) page.insert_text((40, 80), "bhasa pristha") page.insert_text((40, 120), "Sample language page body") document.save(pdf) document.close() publication = tmp_path / "publication.html" publication.write_text( """ समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित लिम्बु भाषा """, encoding="utf-8", ) epaper = tmp_path / "epaper.html" epaper.write_text( f""" २०८३ असार १ सोमवार """, encoding="utf-8", ) audit = audit_gorkhapatra_language_pages( tmp_path / "out", publication_html_paths=[publication], epaper_html_paths=[epaper], download_pdfs=True, render_pages=True, ) assert audit.language_page_hit_count == 1 assert audit.alignments[0].alignment_status == "language_page_found" assert audit.alignments[0].page_hits[0].page_index == 0 assert audit.alignments[0].page_hits[0].rendered_page_path is not None assert Path(audit.alignments[0].page_hits[0].rendered_page_path).exists() def test_prepare_gorkhapatra_language_page_pack_writes_pending_manifest(tmp_path: Path, capsys) -> None: rendered = tmp_path / "page.png" rendered.write_bytes(b"rendered page") audit = tmp_path / "language-pages.json" audit.write_text( json.dumps( { "alignments": [ { "publication_item": { "article_url": "https://gorkhapatraonline.com/news/210290", "image_url": "https://gorkhapatraonline.com/storage/media/365132/6.jpg", "title": "समावेशीकरणको दिशामा नयाँ फड्को : गोरखापत्रद्वारा प्रकाशित मगर भाषा", "language": "मगर", "date_label": "१ असार २०८३, सोमबार", "date_key": "2083-03-01", "source_page": "fixture", }, "epaper": { "viewer_url": "https://epaper.gorkhapatraonline.com/pdf?file=/uploads/file/sample.pdf", "direct_pdf_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf", "source_page": "fixture", "date_label": "२०८३ असार १ सोमवार", "thumbnail_url": None, "downloaded_pdf": None, }, "pdf_path": str(tmp_path / "sample.pdf"), "pdf_sha256": "pdfhash", "page_hits": [ { "page_index": 5, "marker": "efiff k[i7", "text_chars": 120, "text_preview": "efiff k[i7", "rendered_page_path": str(rendered), } ], "alignment_status": "language_page_found", "claim_evidence_eligible": False, "warnings": [], } ] } ), encoding="utf-8", ) exit_code = main( [ "prepare-gorkhapatra-language-page-pack", str(audit), "--out", str(tmp_path / "pack"), ] ) captured = capsys.readouterr() entries = load_manifest(tmp_path / "pack" / "manifests" / "gorkhapatra-language-page-pack.jsonl") reference = json.loads(Path(entries[0].metadata["reference_path"]).read_text(encoding="utf-8")) assert exit_code == 0 assert "samples=1 copied_pages=1 warnings=0" in captured.out assert len(entries) == 1 assert Path(entries[0].image_path).read_bytes() == b"rendered page" assert entries[0].metadata["candidate_language"] == "मगर" assert entries[0].metadata["pdf_page_number"] == 6 assert entries[0].metadata["page_disambiguation_status"] == "candidate_requires_manual_review" assert entries[0].metadata["reference_status"] == "pending_manual_label" assert entries[0].metadata["claim_evidence_eligible"] is False assert "language_page_candidate" in entries[0].metadata["slices"] assert reference["metadata"]["source_kind"] == "gorkhapatra_language_page_candidate" def test_prepare_gorkhapatra_language_page_pack_records_missing_rendered_pages(tmp_path: Path) -> None: audit = tmp_path / "language-pages.json" audit.write_text( json.dumps( { "alignments": [ { "publication_item": {"language": "मगर", "date_key": "2083-03-01"}, "page_hits": [{"page_index": 5, "marker": "efiff k[i7"}], } ] } ), encoding="utf-8", ) summary = prepare_gorkhapatra_language_page_pack(audit, tmp_path / "pack") assert summary.sample_count == 0 assert summary.warnings == ["alignment 1 page hit 1 has no rendered_page_path"] def test_prepare_gorkhapatra_language_page_review_writes_manual_review_artifacts(tmp_path: Path, capsys) -> None: image_one = tmp_path / "page-one.png" image_two = tmp_path / "page-two.png" image_one.write_bytes(b"candidate one") image_two.write_bytes(b"candidate two") ref_one = tmp_path / "page-one.ref.json" ref_two = tmp_path / "page-two.ref.json" ref_one.write_text("{}", encoding="utf-8") ref_two.write_text("{}", encoding="utf-8") manifest = tmp_path / "manifest.jsonl" write_manifest( [ ManifestEntry( sample_id="gorkhapatra-language-page-0001-01", dataset="fixture", split="eval", image_path=str(image_one), text="", sha256=sha256_file(image_one), metadata={ "source_kind": "language_page_candidate", "candidate_language": "संस्कृत", "date_label": "१ असार २०८३, सोमबार", "date_key": "2083-03-01", "pdf_page_index": 5, "pdf_page_number": 6, "reference_path": str(ref_one), "article_url": "https://gorkhapatraonline.com/news/1", "source_image_url": "https://gorkhapatraonline.com/storage/1.jpg", "source_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf", "pdf_path": str(tmp_path / "sample.pdf"), "pdf_sha256": "pdfhash", "language_page_marker": "efiff k[i7", "alignment_status": "language_page_found", "page_disambiguation_status": "candidate_requires_manual_review", "reference_status": "pending_manual_label", "claim_evidence_eligible": False, }, ), ManifestEntry( sample_id="gorkhapatra-language-page-0002-01", dataset="fixture", split="eval", image_path=str(image_two), text="", sha256=sha256_file(image_two), metadata={ "source_kind": "language_page_candidate", "candidate_language": "मगर", "date_key": "2083-03-01", "pdf_page_index": 5, "pdf_page_number": 6, "reference_path": str(ref_two), "source_url": "https://epaper.gorkhapatraonline.com/uploads/file/sample.pdf", "language_page_marker": "efiff k[i7", "alignment_status": "language_page_found", "page_disambiguation_status": "candidate_requires_manual_review", "reference_status": "pending_manual_label", "claim_evidence_eligible": False, }, ), ], manifest, ) exit_code = main( [ "prepare-gorkhapatra-language-page-review", str(manifest), "--out", str(tmp_path / "review"), ] ) captured = capsys.readouterr() payload = json.loads((tmp_path / "review" / "gorkhapatra-language-page-review.json").read_text(encoding="utf-8")) with (tmp_path / "review" / "gorkhapatra-language-page-review.csv").open("r", encoding="utf-8", newline="") as handle: csv_rows = list(csv.DictReader(handle)) markdown = (tmp_path / "review" / "gorkhapatra-language-page-review.md").read_text(encoding="utf-8") assert exit_code == 0 assert "samples=2 pending_review=2 claim_eligible=0 warnings=0" in captured.out assert payload["summary"]["sample_count"] == 2 assert payload["summary"]["page_candidate_counts"] == {"2083-03-01:page-6": 2} assert payload["rows"][0]["review_decision"] == "" assert payload["rows"][0]["claim_evidence_eligible"] is False assert csv_rows[1]["candidate_language"] == "मगर" assert csv_rows[1]["reviewer"] == "" assert "Rows remain non-claim-ready" in markdown