ocr-tech-k2 / src /ocrtech /source_audit.py
prosperious's picture
Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified
Raw History Blame Contribute Delete
67.3 kB
"""Source audits for real-world validation data.""" from __future__ import annotations import csv import json import re import shutil from dataclasses import dataclass, field from html.parser import HTMLParser from pathlib import Path from typing import Any from urllib.parse import parse_qs, unquote, urljoin, urlparse from urllib.request import Request, urlopen from .errors import DataValidationError from .manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest DEFAULT_GORKHAPATRA_CATEGORY_URLS = ( "https://gorkhapatraonline.com/categories/new-nepal", "https://gorkhapatraonline.com/categories/new-nepal?page=2", ) DEFAULT_GORKHAPATRA_EPAPER_URL = ( "https://epaper.gorkhapatraonline.com/epaper-list?" "pdfdaterange1=2026-06-09&pdfdaterange2=2026-06-16&slug=gorkhapatra" ) GORKHAPATRA_ONLINE_BASE = "https://gorkhapatraonline.com" GORKHAPATRA_EPAPER_BASE = "https://epaper.gorkhapatraonline.com" DEFAULT_NAYA_NEPAL_PUBLICATION_URL = "https://gorkhapatraonline.com/publications/naya-nepal" LANGUAGE_PAGE_MARKERS = ("भाषा पृष्ठ", "efiff k[i7", "bhasa pristha", "bhasa prishtha") NEPALI_MONTHS = { "बैशाख": 1, "वैशाख": 1, "जेठ": 2, "असार": 3, "साउन": 4, "भदौ": 5, "भाद्र": 5, "असोज": 6, "कार्तिक": 7, "मंसिर": 8, "मङ्सिर": 8, "पुस": 9, "पुष": 9, "माघ": 10, "फागुन": 11, "चैत": 12, } @dataclass(slots=True) class DownloadedArtifact: url: str path: str sha256: str size_bytes: int kind: str warnings: list[str] = field(default_factory=list) pdf_audit: dict[str, Any] | None = None def to_dict(self) -> dict[str, Any]: payload = { "url": self.url, "path": self.path, "sha256": self.sha256, "size_bytes": self.size_bytes, "kind": self.kind, "warnings": self.warnings, } if self.pdf_audit is not None: payload["pdf_audit"] = self.pdf_audit return payload @dataclass(slots=True) class GorkhapatraArticleAsset: article_url: str | None image_url: str | None title: str language: str | None source_page: str downloaded_image: DownloadedArtifact | None = None def to_dict(self) -> dict[str, Any]: return { "article_url": self.article_url, "image_url": self.image_url, "title": self.title, "language": self.language, "source_page": self.source_page, "downloaded_image": self.downloaded_image.to_dict() if self.downloaded_image else None, } @dataclass(slots=True) class GorkhapatraEpaperAsset: viewer_url: str direct_pdf_url: str source_page: str date_label: str | None = None thumbnail_url: str | None = None downloaded_pdf: DownloadedArtifact | None = None def to_dict(self) -> dict[str, Any]: return { "viewer_url": self.viewer_url, "direct_pdf_url": self.direct_pdf_url, "source_page": self.source_page, "date_label": self.date_label, "thumbnail_url": self.thumbnail_url, "downloaded_pdf": self.downloaded_pdf.to_dict() if self.downloaded_pdf else None, } @dataclass(slots=True) class GorkhapatraSourceAudit: category_sources: list[str] epaper_sources: list[str] article_count: int epaper_count: int language_counts: dict[str, int] downloaded_count: int articles: list[GorkhapatraArticleAsset] epapers: list[GorkhapatraEpaperAsset] warnings: list[str] = field(default_factory=list) summary_json_path: str | None = None summary_md_path: str | None = None def to_dict(self) -> dict[str, Any]: return { "category_sources": self.category_sources, "epaper_sources": self.epaper_sources, "article_count": self.article_count, "epaper_count": self.epaper_count, "language_counts": self.language_counts, "downloaded_count": self.downloaded_count, "warnings": self.warnings, "articles": [article.to_dict() for article in self.articles], "epapers": [epaper.to_dict() for epaper in self.epapers], "summary_json_path": self.summary_json_path, "summary_md_path": self.summary_md_path, } @dataclass(slots=True) class GorkhapatraPackSummary: manifest_path: str sample_count: int article_image_count: int epaper_page_count: int language_counts: dict[str, int] reference_status_counts: dict[str, int] warnings: list[str] = field(default_factory=list) summary_json_path: str | None = None summary_md_path: str | None = None def to_dict(self) -> dict[str, Any]: return { "manifest_path": self.manifest_path, "sample_count": self.sample_count, "article_image_count": self.article_image_count, "epaper_page_count": self.epaper_page_count, "language_counts": self.language_counts, "reference_status_counts": self.reference_status_counts, "warnings": self.warnings, "summary_json_path": self.summary_json_path, "summary_md_path": self.summary_md_path, } @dataclass(slots=True) class NayaNepalPublicationItem: article_url: str | None image_url: str | None title: str language: str | None date_label: str | None date_key: str | None source_page: str def to_dict(self) -> dict[str, Any]: return { "article_url": self.article_url, "image_url": self.image_url, "title": self.title, "language": self.language, "date_label": self.date_label, "date_key": self.date_key, "source_page": self.source_page, } @dataclass(slots=True) class GorkhapatraLanguagePageHit: page_index: int marker: str text_chars: int text_preview: str rendered_page_path: str | None = None def to_dict(self) -> dict[str, Any]: return { "page_index": self.page_index, "marker": self.marker, "text_chars": self.text_chars, "text_preview": self.text_preview, "rendered_page_path": self.rendered_page_path, } @dataclass(slots=True) class GorkhapatraLanguagePageAlignment: publication_item: NayaNepalPublicationItem epaper: GorkhapatraEpaperAsset | None pdf_path: str | None pdf_sha256: str | None page_hits: list[GorkhapatraLanguagePageHit] alignment_status: str claim_evidence_eligible: bool warnings: list[str] = field(default_factory=list) def to_dict(self) -> dict[str, Any]: return { "publication_item": self.publication_item.to_dict(), "epaper": self.epaper.to_dict() if self.epaper else None, "pdf_path": self.pdf_path, "pdf_sha256": self.pdf_sha256, "page_hits": [hit.to_dict() for hit in self.page_hits], "alignment_status": self.alignment_status, "claim_evidence_eligible": self.claim_evidence_eligible, "warnings": self.warnings, } @dataclass(slots=True) class GorkhapatraLanguagePageAudit: publication_sources: list[str] epaper_sources: list[str] publication_item_count: int aligned_count: int language_page_hit_count: int status_counts: dict[str, int] language_counts: dict[str, int] alignments: list[GorkhapatraLanguagePageAlignment] warnings: list[str] = field(default_factory=list) summary_json_path: str | None = None summary_md_path: str | None = None def to_dict(self) -> dict[str, Any]: return { "publication_sources": self.publication_sources, "epaper_sources": self.epaper_sources, "publication_item_count": self.publication_item_count, "aligned_count": self.aligned_count, "language_page_hit_count": self.language_page_hit_count, "status_counts": self.status_counts, "language_counts": self.language_counts, "warnings": self.warnings, "summary_json_path": self.summary_json_path, "summary_md_path": self.summary_md_path, "alignments": [alignment.to_dict() for alignment in self.alignments], } @dataclass(slots=True) class GorkhapatraLanguagePagePackSummary: manifest_path: str sample_count: int copied_page_count: int language_counts: dict[str, int] reference_status_counts: dict[str, int] warnings: list[str] = field(default_factory=list) summary_json_path: str | None = None summary_md_path: str | None = None def to_dict(self) -> dict[str, Any]: return { "manifest_path": self.manifest_path, "sample_count": self.sample_count, "copied_page_count": self.copied_page_count, "language_counts": self.language_counts, "reference_status_counts": self.reference_status_counts, "warnings": self.warnings, "summary_json_path": self.summary_json_path, "summary_md_path": self.summary_md_path, } @dataclass(slots=True) class GorkhapatraLanguagePageReviewSummary: manifest_path: str review_json_path: str review_csv_path: str review_md_path: str sample_count: int pending_review_count: int claim_evidence_eligible_count: int language_counts: dict[str, int] page_candidate_counts: dict[str, int] warnings: list[str] = field(default_factory=list) def to_dict(self) -> dict[str, Any]: return { "manifest_path": self.manifest_path, "review_json_path": self.review_json_path, "review_csv_path": self.review_csv_path, "review_md_path": self.review_md_path, "sample_count": self.sample_count, "pending_review_count": self.pending_review_count, "claim_evidence_eligible_count": self.claim_evidence_eligible_count, "language_counts": self.language_counts, "page_candidate_counts": self.page_candidate_counts, "warnings": self.warnings, } class _GorkhapatraCategoryParser(HTMLParser): def __init__(self, *, base_url: str, source_page: str) -> None: super().__init__(convert_charrefs=True) self.base_url = base_url self.source_page = source_page self.assets: list[GorkhapatraArticleAsset] = [] self._href_stack: list[str | None] = [] self._link_text_stack: list[list[str]] = [] self._title_links: dict[str, str] = {} def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: attr = dict(attrs) if tag == "a": href = attr.get("href") self._href_stack.append(urljoin(self.base_url, href) if href else None) self._link_text_stack.append([]) return if tag != "img": return alt = (attr.get("alt") or "").strip() if "भाषा" not in alt: return src = attr.get("src") article_url = self._nearest_article_url() self.assets.append( GorkhapatraArticleAsset( article_url=article_url, image_url=urljoin(self.base_url, src) if src else None, title=alt, language=_extract_language_label(alt), source_page=self.source_page, ) ) def handle_endtag(self, tag: str) -> None: if tag == "a" and self._href_stack: href = self._href_stack.pop() text_chunks = self._link_text_stack.pop() if self._link_text_stack else [] text = " ".join(" ".join(text_chunks).split()) if href and re.search(r"/news/\d+", href) and "भाषा" in text: self._title_links[text] = href def handle_data(self, data: str) -> None: if self._link_text_stack: self._link_text_stack[-1].append(data) def _nearest_article_url(self) -> str | None: for href in reversed(self._href_stack): if href and re.search(r"/news/\d+", href): return href return None def finalize_assets(self) -> list[GorkhapatraArticleAsset]: for asset in self.assets: if asset.article_url is None: asset.article_url = self._title_links.get(" ".join(asset.title.split())) return self.assets class _GorkhapatraEpaperParser(HTMLParser): def __init__(self, *, base_url: str, source_page: str) -> None: super().__init__(convert_charrefs=True) self.base_url = base_url self.source_page = source_page self.assets: list[GorkhapatraEpaperAsset] = [] self._current_viewer: str | None = None self._last_thumbnail: str | None = None self._capture_date = False def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: attr = dict(attrs) if tag == "a": href = attr.get("href") if href and "/pdf?file=" in href: self._current_viewer = urljoin(self.base_url, href) return if tag == "img": src = attr.get("src") if src and "/uploads/image/" in src: self._last_thumbnail = urljoin(self.base_url, src) return if tag == "span" and "date" in (attr.get("class") or ""): self._capture_date = True def handle_data(self, data: str) -> None: if not self._capture_date or not self._current_viewer: return date_label = " ".join(data.split()) direct_pdf = _direct_epaper_pdf_url(self._current_viewer) self.assets.append( GorkhapatraEpaperAsset( viewer_url=self._current_viewer, direct_pdf_url=direct_pdf, source_page=self.source_page, date_label=date_label or None, thumbnail_url=self._last_thumbnail, ) ) self._current_viewer = None self._last_thumbnail = None self._capture_date = False def handle_endtag(self, tag: str) -> None: if tag == "span": self._capture_date = False class _NayaNepalPublicationParser(HTMLParser): def __init__(self, *, base_url: str, source_page: str) -> None: super().__init__(convert_charrefs=True) self.base_url = base_url self.source_page = source_page self.items: list[NayaNepalPublicationItem] = [] self._href_stack: list[str | None] = [] self._current_index: int | None = None self._capture_title = False def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None: attr = dict(attrs) if tag == "a": href = attr.get("href") self._href_stack.append(urljoin(self.base_url, href) if href else None) return if tag == "img": alt = (attr.get("alt") or "").strip() if "भाषा" not in alt: return src = attr.get("src") self.items.append( NayaNepalPublicationItem( article_url=self._nearest_article_url(), image_url=urljoin(self.base_url, src) if src else None, title=alt, language=_extract_language_label(alt), date_label=None, date_key=None, source_page=self.source_page, ) ) self._current_index = len(self.items) - 1 return if tag in {"h2", "h3"} and self._current_index is not None: self._capture_title = True def handle_endtag(self, tag: str) -> None: if tag == "a" and self._href_stack: self._href_stack.pop() if tag in {"h2", "h3"}: self._capture_title = False def handle_data(self, data: str) -> None: if self._current_index is None: return text = " ".join(data.split()) if not text: return item = self.items[self._current_index] date_key = _nepali_date_key(text) if date_key and item.date_key is None: item.date_label = _clean_date_label(text) item.date_key = date_key if self._capture_title and "भाषा" in text: item.title = text item.language = _extract_language_label(text) def _nearest_article_url(self) -> str | None: for href in reversed(self._href_stack): if href and re.search(r"/news/\d+", href): return href return None def audit_gorkhapatra_source( output_dir: str | Path, *, category_urls: list[str] | None = None, epaper_urls: list[str] | None = None, category_html_paths: list[str | Path] | None = None, epaper_html_paths: list[str | Path] | None = None, download_assets: bool = False, max_article_images: int = 0, max_epaper_pdfs: int = 0, timeout_seconds: float = 30.0, ) -> GorkhapatraSourceAudit: out = Path(output_dir) out.mkdir(parents=True, exist_ok=True) warnings: list[str] = [] category_sources = list(category_urls or ()) epaper_sources = list(epaper_urls or ()) category_inputs = _read_html_inputs(category_sources, category_html_paths or [], timeout_seconds=timeout_seconds, output_dir=out, prefix="category") epaper_inputs = _read_html_inputs(epaper_sources, epaper_html_paths or [], timeout_seconds=timeout_seconds, output_dir=out, prefix="epaper") articles: list[GorkhapatraArticleAsset] = [] for source_name, html_text in category_inputs: parser = _GorkhapatraCategoryParser(base_url=GORKHAPATRA_ONLINE_BASE, source_page=source_name) parser.feed(html_text) articles.extend(parser.finalize_assets()) epapers: list[GorkhapatraEpaperAsset] = [] for source_name, html_text in epaper_inputs: parser = _GorkhapatraEpaperParser(base_url=GORKHAPATRA_EPAPER_BASE, source_page=source_name) parser.feed(html_text) epapers.extend(parser.assets) articles = _dedupe_articles(articles) epapers = _dedupe_epapers(epapers) if download_assets: _download_article_images(articles, out, max_count=max_article_images, timeout_seconds=timeout_seconds, warnings=warnings) _download_epaper_pdfs(epapers, out, max_count=max_epaper_pdfs, timeout_seconds=timeout_seconds, warnings=warnings) language_counts: dict[str, int] = {} for article in articles: if article.language: language_counts[article.language] = language_counts.get(article.language, 0) + 1 downloaded_count = sum(1 for article in articles if article.downloaded_image) + sum(1 for epaper in epapers if epaper.downloaded_pdf) audit = GorkhapatraSourceAudit( category_sources=[source for source, _ in category_inputs], epaper_sources=[source for source, _ in epaper_inputs], article_count=len(articles), epaper_count=len(epapers), language_counts=dict(sorted(language_counts.items())), downloaded_count=downloaded_count, articles=articles, epapers=epapers, warnings=warnings, ) _write_gorkhapatra_audit(audit, out) return audit def prepare_gorkhapatra_pack( audit_json_path: str | Path, output_dir: str | Path, *, include_article_images: bool = True, include_epaper_pages: bool = True, max_article_images: int = 0, max_epaper_pdfs: int = 0, max_pages_per_pdf: int = 2, split: str = "eval", dataset_name: str = "gorkhapatra-naya-nepal", ) -> GorkhapatraPackSummary: audit_path = Path(audit_json_path) if not audit_path.is_file(): raise DataValidationError(f"Gorkhapatra source audit JSON does not exist: {audit_path}") if max_pages_per_pdf < 1: raise DataValidationError("max_pages_per_pdf must be at least 1") payload = json.loads(audit_path.read_text(encoding="utf-8")) out = Path(output_dir) inputs_dir = out / "inputs" refs_dir = out / "references" manifests_dir = out / "manifests" inputs_dir.mkdir(parents=True, exist_ok=True) refs_dir.mkdir(parents=True, exist_ok=True) manifests_dir.mkdir(parents=True, exist_ok=True) entries: list[ManifestEntry] = [] warnings: list[str] = [] article_count = 0 epaper_page_count = 0 if include_article_images: for article_index, article in enumerate(payload.get("articles") or [], start=1): if max_article_images and article_count >= max_article_images: break downloaded = article.get("downloaded_image") if isinstance(article, dict) else None if not isinstance(downloaded, dict) or not downloaded.get("path"): continue source_path = _resolve_audit_artifact_path(str(downloaded["path"]), audit_path) if not source_path.is_file(): warnings.append(f"missing downloaded article image for article {article_index}: {source_path}") continue sample_id = f"gorkhapatra-article-{article_index:04d}" target_path = inputs_dir / f"{sample_id}{source_path.suffix.lower() or '.jpg'}" shutil.copy2(source_path, target_path) reference_path = refs_dir / f"{sample_id}.ref.json" language = _clean_optional_str(article.get("language")) source_url = _clean_optional_str(article.get("article_url")) or _clean_optional_str(article.get("image_url")) _write_pending_reference( reference_path, sample_id=sample_id, source_url=source_url, language=language, source_kind="gorkhapatra_article_image", ) image_hash = sha256_file(target_path) entries.append( ManifestEntry( sample_id=sample_id, dataset=dataset_name, split=split, image_path=str(target_path), text="", sha256=image_hash, metadata={ "benchmark_name": "gorkhapatra-naya-nepal", "source": "gorkhapatra", "source_kind": "article_image", "source_url": source_url, "source_image_url": article.get("image_url"), "language": language, "script": "unknown", "input_format": "image", "document_type": "article_image", "reference_path": str(reference_path), "reference_status": "pending_manual_label", "claim_evidence_eligible": False, "slices": _gorkhapatra_slices(language, "article_image"), "text_sha256": sha256_text(""), "sample_sha256": sha256_text(f"{image_hash}\n"), }, ) ) article_count += 1 if include_epaper_pages: epapers = payload.get("epapers") or [] rendered_pdf_count = 0 for epaper_index, epaper in enumerate(epapers, start=1): if max_epaper_pdfs and rendered_pdf_count >= max_epaper_pdfs: break downloaded = epaper.get("downloaded_pdf") if isinstance(epaper, dict) else None if not isinstance(downloaded, dict) or not downloaded.get("path"): continue source_path = _resolve_audit_artifact_path(str(downloaded["path"]), audit_path) if not source_path.is_file(): warnings.append(f"missing downloaded epaper PDF for epaper {epaper_index}: {source_path}") continue try: rendered_pages = _render_pdf_pages(source_path, inputs_dir, epaper_index=epaper_index, max_pages=max_pages_per_pdf) except DataValidationError as exc: warnings.append(str(exc)) continue rendered_pdf_count += 1 for page_index, target_path in rendered_pages: sample_id = f"gorkhapatra-epaper-{epaper_index:04d}-page-{page_index + 1:04d}" reference_path = refs_dir / f"{sample_id}.ref.json" source_url = _clean_optional_str(epaper.get("direct_pdf_url")) or _clean_optional_str(epaper.get("viewer_url")) _write_pending_reference( reference_path, sample_id=sample_id, source_url=source_url, language=None, source_kind="gorkhapatra_epaper_page", ) image_hash = sha256_file(target_path) entries.append( ManifestEntry( sample_id=sample_id, dataset=dataset_name, split=split, image_path=str(target_path), text="", sha256=image_hash, metadata={ "benchmark_name": "gorkhapatra-naya-nepal", "source": "gorkhapatra", "source_kind": "epaper_page", "source_url": source_url, "viewer_url": epaper.get("viewer_url"), "date_label": epaper.get("date_label"), "pdf_sha256": downloaded.get("sha256"), "pdf_page_index": page_index, "language": None, "script": "unknown", "input_format": "image", "document_type": "epaper_page", "reference_path": str(reference_path), "reference_status": "pending_manual_label", "claim_evidence_eligible": False, "slices": _gorkhapatra_slices(None, "epaper_page"), "text_sha256": sha256_text(""), "sample_sha256": sha256_text(f"{image_hash}\n"), }, ) ) epaper_page_count += 1 manifest_path = manifests_dir / "gorkhapatra-pack.jsonl" write_manifest(entries, manifest_path) summary = _gorkhapatra_pack_summary( manifest_path, entries, article_image_count=article_count, epaper_page_count=epaper_page_count, warnings=warnings, ) _write_gorkhapatra_pack_summary(summary, out) return summary def audit_gorkhapatra_language_pages( output_dir: str | Path, *, publication_urls: list[str] | None = None, epaper_urls: list[str] | None = None, publication_html_paths: list[str | Path] | None = None, epaper_html_paths: list[str | Path] | None = None, download_pdfs: bool = False, render_pages: bool = False, max_publication_items: int = 0, max_epaper_pdfs: int = 0, timeout_seconds: float = 30.0, ) -> GorkhapatraLanguagePageAudit: out = Path(output_dir) out.mkdir(parents=True, exist_ok=True) warnings: list[str] = [] publication_sources = list(publication_urls or ()) epaper_sources = list(epaper_urls or ()) publication_inputs = _read_html_inputs( publication_sources, publication_html_paths or [], timeout_seconds=timeout_seconds, output_dir=out, prefix="naya-nepal-publication", ) epaper_inputs = _read_html_inputs(epaper_sources, epaper_html_paths or [], timeout_seconds=timeout_seconds, output_dir=out, prefix="epaper") publication_items: list[NayaNepalPublicationItem] = [] for source_name, html_text in publication_inputs: parser = _NayaNepalPublicationParser(base_url=GORKHAPATRA_ONLINE_BASE, source_page=source_name) parser.feed(html_text) publication_items.extend(parser.items) publication_items = _dedupe_publication_items(publication_items) if max_publication_items: publication_items = publication_items[:max_publication_items] epapers: list[GorkhapatraEpaperAsset] = [] for source_name, html_text in epaper_inputs: parser = _GorkhapatraEpaperParser(base_url=GORKHAPATRA_EPAPER_BASE, source_page=source_name) parser.feed(html_text) epapers.extend(parser.assets) epapers = _dedupe_epapers(epapers) epapers_by_date = {_nepali_date_key(epaper.date_label or ""): epaper for epaper in epapers if _nepali_date_key(epaper.date_label or "")} alignments: list[GorkhapatraLanguagePageAlignment] = [] downloaded_pdf_count = 0 pdf_cache: dict[str, DownloadedArtifact] = {} for index, item in enumerate(publication_items, start=1): item_warnings: list[str] = [] if not item.date_key: alignments.append(_language_page_alignment(item, None, None, None, [], "missing_publication_date", item_warnings)) continue epaper = epapers_by_date.get(item.date_key) if epaper is None: alignments.append(_language_page_alignment(item, None, None, None, [], "no_matching_epaper_date", item_warnings)) continue if not download_pdfs: alignments.append(_language_page_alignment(item, epaper, None, None, [], "matched_epaper_not_downloaded", item_warnings)) continue if max_epaper_pdfs and downloaded_pdf_count >= max_epaper_pdfs and epaper.direct_pdf_url not in pdf_cache: alignments.append(_language_page_alignment(item, epaper, None, None, [], "matched_epaper_not_downloaded_limit", item_warnings)) continue try: artifact = pdf_cache.get(epaper.direct_pdf_url) if artifact is None: target = out / "epapers" / f"epaper-{item.date_key}.pdf" artifact = _download_url(epaper.direct_pdf_url, target, timeout_seconds=timeout_seconds) artifact.pdf_audit = _audit_pdf_with_pymupdf(target) pdf_cache[epaper.direct_pdf_url] = artifact downloaded_pdf_count += 1 epaper.downloaded_pdf = artifact hits = _find_language_page_hits(Path(artifact.path), out / "language-pages", render_pages=render_pages) status = "language_page_found" if hits else "matched_epaper_no_language_page_marker" alignments.append(_language_page_alignment(item, epaper, artifact.path, artifact.sha256, hits, status, item_warnings)) except Exception as exc: # noqa: BLE001 - recorded as provenance warning, not ignored. warning = f"{type(exc).__name__}: {exc}" item_warnings.append(warning) warnings.append(f"failed to inspect {epaper.direct_pdf_url}: {warning}") alignments.append(_language_page_alignment(item, epaper, None, None, [], "pdf_inspection_failed", item_warnings)) audit = _gorkhapatra_language_page_audit( publication_sources=[source for source, _ in publication_inputs], epaper_sources=[source for source, _ in epaper_inputs], publication_items=publication_items, alignments=alignments, warnings=warnings, ) _write_gorkhapatra_language_page_audit(audit, out) return audit def prepare_gorkhapatra_language_page_pack( language_page_audit_json_path: str | Path, output_dir: str | Path, *, split: str = "eval", dataset_name: str = "gorkhapatra-language-pages", max_samples: int = 0, ) -> GorkhapatraLanguagePagePackSummary: audit_path = Path(language_page_audit_json_path) if not audit_path.is_file(): raise DataValidationError(f"Gorkhapatra language-page audit JSON does not exist: {audit_path}") payload = json.loads(audit_path.read_text(encoding="utf-8")) out = Path(output_dir) inputs_dir = out / "inputs" refs_dir = out / "references" manifests_dir = out / "manifests" inputs_dir.mkdir(parents=True, exist_ok=True) refs_dir.mkdir(parents=True, exist_ok=True) manifests_dir.mkdir(parents=True, exist_ok=True) entries: list[ManifestEntry] = [] warnings: list[str] = [] copied_count = 0 for alignment_index, alignment in enumerate(payload.get("alignments") or [], start=1): if max_samples and len(entries) >= max_samples: break if not isinstance(alignment, dict): continue publication_item = alignment.get("publication_item") if not isinstance(publication_item, dict): warnings.append(f"alignment {alignment_index} missing publication_item") continue language = _clean_optional_str(publication_item.get("language")) date_key = _clean_optional_str(publication_item.get("date_key")) or "unknown-date" page_hits = alignment.get("page_hits") if not isinstance(page_hits, list) or not page_hits: warnings.append(f"alignment {alignment_index} has no rendered page hits") continue for hit_index, hit in enumerate(page_hits, start=1): if max_samples and len(entries) >= max_samples: break if not isinstance(hit, dict): continue rendered_path_value = _clean_optional_str(hit.get("rendered_page_path")) if not rendered_path_value: warnings.append(f"alignment {alignment_index} page hit {hit_index} has no rendered_page_path") continue source_path = _resolve_audit_artifact_path(rendered_path_value, audit_path) if not source_path.is_file(): warnings.append(f"alignment {alignment_index} page hit {hit_index} missing rendered page: {source_path}") continue page_index = int(hit.get("page_index") or 0) sample_id = f"gorkhapatra-language-page-{alignment_index:04d}-{hit_index:02d}" target_path = inputs_dir / f"{sample_id}{source_path.suffix.lower() or '.png'}" shutil.copy2(source_path, target_path) reference_path = refs_dir / f"{sample_id}.ref.json" source_url = _alignment_source_url(alignment) _write_pending_reference( reference_path, sample_id=sample_id, source_url=source_url, language=language, source_kind="gorkhapatra_language_page_candidate", ) image_hash = sha256_file(target_path) entries.append( ManifestEntry( sample_id=sample_id, dataset=dataset_name, split=split, image_path=str(target_path), text="", sha256=image_hash, metadata={ "benchmark_name": "gorkhapatra-naya-nepal-language-pages", "source": "gorkhapatra", "source_kind": "language_page_candidate", "source_url": source_url, "article_url": publication_item.get("article_url"), "source_image_url": publication_item.get("image_url"), "candidate_language": language, "language": language, "date_label": publication_item.get("date_label"), "date_key": date_key, "pdf_path": alignment.get("pdf_path"), "pdf_sha256": alignment.get("pdf_sha256"), "pdf_page_index": page_index, "pdf_page_number": page_index + 1, "language_page_marker": hit.get("marker"), "alignment_status": alignment.get("alignment_status"), "page_disambiguation_status": "candidate_requires_manual_review", "script": "unknown", "input_format": "image", "document_type": "language_page_candidate", "reference_path": str(reference_path), "reference_status": "pending_manual_label", "claim_evidence_eligible": False, "slices": _gorkhapatra_slices(language, "language_page_candidate"), "text_sha256": sha256_text(""), "sample_sha256": sha256_text(f"{image_hash}\n{alignment.get('pdf_sha256') or ''}\n{page_index}\n"), }, ) ) copied_count += 1 manifest_path = manifests_dir / "gorkhapatra-language-page-pack.jsonl" write_manifest(entries, manifest_path) summary = _gorkhapatra_language_page_pack_summary( manifest_path, entries, copied_page_count=copied_count, warnings=warnings, ) _write_gorkhapatra_language_page_pack_summary(summary, out) return summary def prepare_gorkhapatra_language_page_review( manifest_path: str | Path, output_dir: str | Path, ) -> GorkhapatraLanguagePageReviewSummary: source_manifest = Path(manifest_path) if not source_manifest.is_file(): raise DataValidationError(f"Gorkhapatra language-page pack manifest does not exist: {source_manifest}") entries = load_manifest(source_manifest) out = Path(output_dir) out.mkdir(parents=True, exist_ok=True) rows: list[dict[str, Any]] = [] warnings: list[str] = [] language_counts: dict[str, int] = {} page_candidate_counts: dict[str, int] = {} for entry in entries: metadata = entry.metadata or {} source_kind = metadata.get("source_kind") if source_kind != "language_page_candidate": warnings.append(f"{entry.sample_id} source_kind is not language_page_candidate: {source_kind}") candidate_language = _clean_optional_str(metadata.get("candidate_language")) or _clean_optional_str(metadata.get("language")) if candidate_language: language_counts[candidate_language] = language_counts.get(candidate_language, 0) + 1 date_key = _clean_optional_str(metadata.get("date_key")) or "unknown-date" page_number = metadata.get("pdf_page_number") page_key = f"{date_key}:page-{page_number if page_number is not None else 'unknown'}" page_candidate_counts[page_key] = page_candidate_counts.get(page_key, 0) + 1 rows.append( { "sample_id": entry.sample_id, "candidate_language": candidate_language, "date_label": metadata.get("date_label"), "date_key": metadata.get("date_key"), "pdf_page_index": metadata.get("pdf_page_index"), "pdf_page_number": metadata.get("pdf_page_number"), "image_path": entry.image_path, "reference_path": metadata.get("reference_path"), "article_url": metadata.get("article_url"), "source_image_url": metadata.get("source_image_url"), "source_url": metadata.get("source_url"), "pdf_path": metadata.get("pdf_path"), "pdf_sha256": metadata.get("pdf_sha256"), "language_page_marker": metadata.get("language_page_marker"), "alignment_status": metadata.get("alignment_status"), "page_disambiguation_status": metadata.get("page_disambiguation_status"), "reference_status": metadata.get("reference_status"), "claim_evidence_eligible": bool(metadata.get("claim_evidence_eligible")), "review_decision": "", "reviewer": "", "reviewed_at": "", "notes": "", } ) json_path = out / "gorkhapatra-language-page-review.json" csv_path = out / "gorkhapatra-language-page-review.csv" md_path = out / "gorkhapatra-language-page-review.md" summary = GorkhapatraLanguagePageReviewSummary( manifest_path=str(source_manifest), review_json_path=str(json_path), review_csv_path=str(csv_path), review_md_path=str(md_path), sample_count=len(rows), pending_review_count=sum(1 for row in rows if row["page_disambiguation_status"] == "candidate_requires_manual_review"), claim_evidence_eligible_count=sum(1 for row in rows if row["claim_evidence_eligible"]), language_counts=dict(sorted(language_counts.items())), page_candidate_counts=dict(sorted(page_candidate_counts.items())), warnings=warnings, ) _write_gorkhapatra_language_page_review(summary, rows) return summary def _read_html_inputs( urls: list[str], paths: list[str | Path], *, timeout_seconds: float, output_dir: Path, prefix: str, ) -> list[tuple[str, str]]: inputs: list[tuple[str, str]] = [] raw_dir = output_dir / "raw" raw_dir.mkdir(parents=True, exist_ok=True) for index, url in enumerate(urls, start=1): text = _fetch_text(url, timeout_seconds=timeout_seconds) raw_path = raw_dir / f"{prefix}-url-{index}.html" raw_path.write_text(text, encoding="utf-8") inputs.append((url, text)) for path in paths: html_path = Path(path) if not html_path.is_file(): raise DataValidationError(f"{prefix} HTML path does not exist: {html_path}") inputs.append((str(html_path), html_path.read_text(encoding="utf-8"))) return inputs def _fetch_text(url: str, *, timeout_seconds: float) -> str: request = Request(url, headers={"User-Agent": "ocrtech-source-audit/0.1"}) with urlopen(request, timeout=timeout_seconds) as response: data = response.read() return data.decode("utf-8", errors="replace") def _download_url(url: str, path: Path, *, timeout_seconds: float) -> DownloadedArtifact: path.parent.mkdir(parents=True, exist_ok=True) request = Request(url, headers={"User-Agent": "ocrtech-source-audit/0.1"}) with urlopen(request, timeout=timeout_seconds) as response: path.write_bytes(response.read()) return DownloadedArtifact(url=url, path=str(path), sha256=sha256_file(path), size_bytes=path.stat().st_size, kind=_artifact_kind(path)) def _download_article_images( articles: list[GorkhapatraArticleAsset], output_dir: Path, *, max_count: int, timeout_seconds: float, warnings: list[str], ) -> None: count = 0 for index, article in enumerate(articles, start=1): if max_count and count >= max_count: return if not article.image_url: continue suffix = _url_suffix(article.image_url, default=".jpg") target = output_dir / "article-images" / f"article-{index:04d}{suffix}" try: article.downloaded_image = _download_url(article.image_url, target, timeout_seconds=timeout_seconds) count += 1 except Exception as exc: # noqa: BLE001 - wrapped as provenance warning, not ignored. warnings.append(f"failed to download article image {article.image_url}: {type(exc).__name__}: {exc}") def _download_epaper_pdfs( epapers: list[GorkhapatraEpaperAsset], output_dir: Path, *, max_count: int, timeout_seconds: float, warnings: list[str], ) -> None: count = 0 for index, epaper in enumerate(epapers, start=1): if max_count and count >= max_count: return target = output_dir / "epapers" / f"epaper-{index:04d}.pdf" try: artifact = _download_url(epaper.direct_pdf_url, target, timeout_seconds=timeout_seconds) artifact.pdf_audit = _audit_pdf_with_pymupdf(target) epaper.downloaded_pdf = artifact count += 1 except Exception as exc: # noqa: BLE001 - wrapped as provenance warning, not ignored. warnings.append(f"failed to download epaper PDF {epaper.direct_pdf_url}: {type(exc).__name__}: {exc}") def _audit_pdf_with_pymupdf(path: Path) -> dict[str, Any]: try: import fitz # type: ignore[import-not-found] except Exception as exc: # noqa: BLE001 - optional dependency. return {"available": False, "error": f"{type(exc).__name__}: {exc}"} with fitz.open(path) as document: pages: list[dict[str, Any]] = [] total_text_chars = 0 for index, page in enumerate(document): text = page.get_text("text") or "" total_text_chars += len(text) if index < 5: pages.append( { "page_index": index, "text_chars": len(text), "text_preview": text[:300], "block_count": len(page.get_text("blocks") or []), "image_count": len(page.get_images(full=True) or []), } ) return { "available": True, "page_count": document.page_count, "total_text_chars": total_text_chars, "first_pages": pages, "text_layer_warning": _pdf_text_layer_warning(pages), } def _pdf_text_layer_warning(pages: list[dict[str, Any]]) -> str | None: preview = "\n".join(str(page.get("text_preview") or "") for page in pages) if not preview.strip(): return "no_text_extracted" devanagari = sum(1 for char in preview if "\u0900" <= char <= "\u097f") ascii_like = sum(1 for char in preview if char.isascii() and char.isalnum()) if ascii_like > devanagari * 5 and devanagari < 20: return "extracted_text_may_be_legacy_encoded_or_mojibake" return None def _find_language_page_hits(pdf_path: Path, output_dir: Path, *, render_pages: bool) -> list[GorkhapatraLanguagePageHit]: try: import fitz # type: ignore[import-not-found] except Exception as exc: # noqa: BLE001 - optional dependency. raise DataValidationError(f"PDF page marker search requires PyMuPDF: {type(exc).__name__}: {exc}") from exc hits: list[GorkhapatraLanguagePageHit] = [] if render_pages: output_dir.mkdir(parents=True, exist_ok=True) with fitz.open(pdf_path) as document: for page_index, page in enumerate(document): text = page.get_text("text") or "" marker = _language_page_marker(text) if marker is None: continue rendered_path: str | None = None if render_pages: target = output_dir / f"{pdf_path.stem}-page-{page_index + 1:04d}.png" pixmap = page.get_pixmap(matrix=fitz.Matrix(2.0, 2.0), alpha=False) pixmap.save(target) rendered_path = str(target) hits.append( GorkhapatraLanguagePageHit( page_index=page_index, marker=marker, text_chars=len(text), text_preview=" ".join(text.split())[:500], rendered_page_path=rendered_path, ) ) return hits def _language_page_marker(text: str) -> str | None: normalized = " ".join(text.split()).lower() for marker in LANGUAGE_PAGE_MARKERS: if marker.lower() in normalized: return marker if "भाषा" in normalized and "पृष्ठ" in normalized: return "भाषा+पृष्ठ" return None def _write_gorkhapatra_audit(audit: GorkhapatraSourceAudit, output_dir: Path) -> None: json_path = output_dir / "gorkhapatra-source-audit.json" md_path = output_dir / "gorkhapatra-source-audit.md" audit.summary_json_path = str(json_path) audit.summary_md_path = str(md_path) json_path.write_text(json.dumps(audit.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") lines = [ "# Gorkhapatra Source Audit", "", f"- article assets: {audit.article_count}", f"- epaper assets: {audit.epaper_count}", f"- downloaded artifacts: {audit.downloaded_count}", f"- warnings: {len(audit.warnings)}", "", "## Languages", "", ] if audit.language_counts: lines.extend(f"- {language}: {count}" for language, count in audit.language_counts.items()) else: lines.append("- none detected") if audit.warnings: lines.extend(["", "## Warnings", ""]) lines.extend(f"- {warning}" for warning in audit.warnings) lines.extend(["", "## Sample Article Assets", ""]) for article in audit.articles[:20]: lines.append(f"- {article.language or 'unknown'}: {article.title} ({article.article_url or 'no article URL'})") lines.extend(["", "## Sample Epaper Assets", ""]) for epaper in audit.epapers[:20]: lines.append(f"- {epaper.date_label or 'unknown date'}: {epaper.direct_pdf_url}") md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") def _write_gorkhapatra_language_page_audit(audit: GorkhapatraLanguagePageAudit, output_dir: Path) -> None: json_path = output_dir / "gorkhapatra-language-pages.json" md_path = output_dir / "gorkhapatra-language-pages.md" audit.summary_json_path = str(json_path) audit.summary_md_path = str(md_path) json_path.write_text(json.dumps(audit.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") lines = [ "# Gorkhapatra Language Page Alignment", "", f"- publication items: {audit.publication_item_count}", f"- aligned to epaper date: {audit.aligned_count}", f"- language page hits: {audit.language_page_hit_count}", f"- warnings: {len(audit.warnings)}", "", "## Status", "", ] lines.extend(f"- `{status}`: {count}" for status, count in audit.status_counts.items()) if audit.status_counts else lines.append("- none") lines.extend(["", "## Languages", ""]) lines.extend(f"- {language}: {count}" for language, count in audit.language_counts.items()) if audit.language_counts else lines.append("- none") lines.extend(["", "## Alignments", ""]) for alignment in audit.alignments[:50]: item = alignment.publication_item page_list = ", ".join(str(hit.page_index + 1) for hit in alignment.page_hits) or "none" lines.append( f"- {item.date_label or 'unknown date'} / {item.language or 'unknown'}: " f"`{alignment.alignment_status}` pages={page_list} title={item.title}" ) if audit.warnings: lines.extend(["", "## Warnings", ""]) lines.extend(f"- {warning}" for warning in audit.warnings) md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") def _direct_epaper_pdf_url(viewer_url: str) -> str: parsed = urlparse(viewer_url) values = parse_qs(parsed.query).get("file") if not values: raise DataValidationError(f"epaper viewer URL does not include file parameter: {viewer_url}") return urljoin(GORKHAPATRA_EPAPER_BASE, unquote(values[0])) def _extract_language_label(title: str) -> str | None: match = re.search(r"प्रकाशित\s+(.+?)\s+भाषा", title) if not match: return None return " ".join(match.group(1).split()) def _nepali_date_key(text: str) -> str | None: normalized = _normalize_nepali_digits(" ".join(text.split())) patterns = ( r"(?P<day>\d{1,2})\s+(?P<month>[^\s,]+)\s+(?P<year>\d{4})", r"(?P<year>\d{4})\s+(?P<month>[^\s,]+)\s+(?P<day>\d{1,2})", ) for pattern in patterns: match = re.search(pattern, normalized) if not match: continue month = NEPALI_MONTHS.get(match.group("month")) if month is None: continue return f"{int(match.group('year')):04d}-{month:02d}-{int(match.group('day')):02d}" return None def _normalize_nepali_digits(text: str) -> str: table = str.maketrans("०१२३४५६७८९", "0123456789") return text.translate(table) def _clean_date_label(text: str) -> str: return " ".join(text.split()).strip() def _dedupe_publication_items(items: list[NayaNepalPublicationItem]) -> list[NayaNepalPublicationItem]: seen: set[tuple[str | None, str | None, str, str | None]] = set() unique: list[NayaNepalPublicationItem] = [] for item in items: key = (item.article_url, item.image_url, item.title, item.date_key) if key in seen: continue seen.add(key) unique.append(item) return unique def _language_page_alignment( item: NayaNepalPublicationItem, epaper: GorkhapatraEpaperAsset | None, pdf_path: str | None, pdf_sha256: str | None, hits: list[GorkhapatraLanguagePageHit], status: str, warnings: list[str], ) -> GorkhapatraLanguagePageAlignment: return GorkhapatraLanguagePageAlignment( publication_item=item, epaper=epaper, pdf_path=pdf_path, pdf_sha256=pdf_sha256, page_hits=hits, alignment_status=status, claim_evidence_eligible=False, warnings=warnings, ) def _alignment_source_url(alignment: dict[str, Any]) -> str | None: epaper = alignment.get("epaper") if isinstance(epaper, dict): return _clean_optional_str(epaper.get("direct_pdf_url")) or _clean_optional_str(epaper.get("viewer_url")) publication_item = alignment.get("publication_item") if isinstance(publication_item, dict): return _clean_optional_str(publication_item.get("article_url")) or _clean_optional_str(publication_item.get("image_url")) return None def _gorkhapatra_language_page_audit( *, publication_sources: list[str], epaper_sources: list[str], publication_items: list[NayaNepalPublicationItem], alignments: list[GorkhapatraLanguagePageAlignment], warnings: list[str], ) -> GorkhapatraLanguagePageAudit: status_counts: dict[str, int] = {} language_counts: dict[str, int] = {} for alignment in alignments: status_counts[alignment.alignment_status] = status_counts.get(alignment.alignment_status, 0) + 1 language = alignment.publication_item.language if language: language_counts[language] = language_counts.get(language, 0) + 1 return GorkhapatraLanguagePageAudit( publication_sources=publication_sources, epaper_sources=epaper_sources, publication_item_count=len(publication_items), aligned_count=sum(1 for alignment in alignments if alignment.epaper is not None), language_page_hit_count=sum(len(alignment.page_hits) for alignment in alignments), status_counts=dict(sorted(status_counts.items())), language_counts=dict(sorted(language_counts.items())), alignments=alignments, warnings=warnings, ) def _gorkhapatra_language_page_pack_summary( manifest_path: Path, entries: list[ManifestEntry], *, copied_page_count: int, warnings: list[str], ) -> GorkhapatraLanguagePagePackSummary: language_counts: dict[str, int] = {} reference_status_counts: dict[str, int] = {} for entry in entries: language = entry.metadata.get("language") if isinstance(language, str) and language: language_counts[language] = language_counts.get(language, 0) + 1 status = str(entry.metadata.get("reference_status") or "unknown") reference_status_counts[status] = reference_status_counts.get(status, 0) + 1 return GorkhapatraLanguagePagePackSummary( manifest_path=str(manifest_path), sample_count=len(entries), copied_page_count=copied_page_count, language_counts=dict(sorted(language_counts.items())), reference_status_counts=dict(sorted(reference_status_counts.items())), warnings=warnings, ) def _dedupe_articles(articles: list[GorkhapatraArticleAsset]) -> list[GorkhapatraArticleAsset]: seen: set[tuple[str | None, str | None, str]] = set() unique: list[GorkhapatraArticleAsset] = [] for article in articles: key = (article.article_url, article.image_url, article.title) if key in seen: continue seen.add(key) unique.append(article) return unique def _dedupe_epapers(epapers: list[GorkhapatraEpaperAsset]) -> list[GorkhapatraEpaperAsset]: seen: set[str] = set() unique: list[GorkhapatraEpaperAsset] = [] for epaper in epapers: if epaper.direct_pdf_url in seen: continue seen.add(epaper.direct_pdf_url) unique.append(epaper) return unique def _artifact_kind(path: Path) -> str: suffix = path.suffix.lower() if suffix == ".pdf": return "pdf" if suffix in {".jpg", ".jpeg", ".png", ".webp"}: return "image" return "file" def _url_suffix(url: str, *, default: str) -> str: suffix = Path(urlparse(url).path).suffix.lower() if suffix in {".jpg", ".jpeg", ".png", ".webp", ".pdf"}: return suffix return default def _resolve_audit_artifact_path(path_value: str, audit_json_path: Path) -> Path: path = Path(path_value) if path.is_absolute(): return path audit_dir = audit_json_path.parent candidate = audit_dir / path if candidate.exists(): return candidate return path def _write_pending_reference( path: Path, *, sample_id: str, source_url: str | None, language: str | None, source_kind: str, ) -> None: payload = { "text": "", "reading_order": [], "tables": [], "figures": [], "metadata": { "sample_id": sample_id, "source_url": source_url, "language": language, "source_kind": source_kind, "reference_status": "pending_manual_label", "claim_evidence_eligible": False, "notes": "Real Gorkhapatra source input. Text/layout reference must be manually verified before claim use.", }, } path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") def _render_pdf_pages(source_pdf: Path, inputs_dir: Path, *, epaper_index: int, max_pages: int) -> list[tuple[int, Path]]: try: import fitz # type: ignore[import-not-found] except Exception as exc: # noqa: BLE001 - optional dependency. raise DataValidationError(f"PDF page rendering requires PyMuPDF: {type(exc).__name__}: {exc}") from exc rendered: list[tuple[int, Path]] = [] with fitz.open(source_pdf) as document: page_count = min(document.page_count, max_pages) for page_index in range(page_count): page = document[page_index] target = inputs_dir / f"gorkhapatra-epaper-{epaper_index:04d}-page-{page_index + 1:04d}.png" pixmap = page.get_pixmap(matrix=fitz.Matrix(2.0, 2.0), alpha=False) pixmap.save(target) rendered.append((page_index, target)) return rendered def _gorkhapatra_slices(language: str | None, source_kind: str) -> list[str]: slices = {"gorkhapatra", "naya_nepal", "real_publication", "image"} if source_kind == "article_image": slices.add("article_image") if source_kind == "epaper_page": slices.update({"epaper", "page"}) if source_kind == "language_page_candidate": slices.update({"epaper", "page", "language_page_candidate", "pending_reference"}) if language: slices.add(f"language:{language}") return sorted(slices) def _gorkhapatra_pack_summary( manifest_path: Path, entries: list[ManifestEntry], *, article_image_count: int, epaper_page_count: int, warnings: list[str], ) -> GorkhapatraPackSummary: language_counts: dict[str, int] = {} reference_status_counts: dict[str, int] = {} for entry in entries: language = entry.metadata.get("language") if isinstance(language, str) and language: language_counts[language] = language_counts.get(language, 0) + 1 status = str(entry.metadata.get("reference_status") or "unknown") reference_status_counts[status] = reference_status_counts.get(status, 0) + 1 return GorkhapatraPackSummary( manifest_path=str(manifest_path), sample_count=len(entries), article_image_count=article_image_count, epaper_page_count=epaper_page_count, language_counts=dict(sorted(language_counts.items())), reference_status_counts=dict(sorted(reference_status_counts.items())), warnings=warnings, ) def _write_gorkhapatra_pack_summary(summary: GorkhapatraPackSummary, output_dir: Path) -> None: json_path = output_dir / "gorkhapatra-pack.json" md_path = output_dir / "gorkhapatra-pack.md" summary.summary_json_path = str(json_path) summary.summary_md_path = str(md_path) json_path.write_text(json.dumps(summary.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") lines = [ "# Gorkhapatra Validation Pack", "", f"- manifest: `{summary.manifest_path}`", f"- samples: {summary.sample_count}", f"- article images: {summary.article_image_count}", f"- epaper pages: {summary.epaper_page_count}", "", "## Reference Status", "", ] lines.extend(f"- `{key}`: {value}" for key, value in summary.reference_status_counts.items()) if summary.reference_status_counts else lines.append("- none") lines.extend(["", "## Languages", ""]) lines.extend(f"- {language}: {count}" for language, count in summary.language_counts.items()) if summary.language_counts else lines.append("- none") lines.extend(["", "## Warnings", ""]) lines.extend(f"- {warning}" for warning in summary.warnings) if summary.warnings else lines.append("- none") md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") def _write_gorkhapatra_language_page_pack_summary(summary: GorkhapatraLanguagePagePackSummary, output_dir: Path) -> None: json_path = output_dir / "gorkhapatra-language-page-pack.json" md_path = output_dir / "gorkhapatra-language-page-pack.md" summary.summary_json_path = str(json_path) summary.summary_md_path = str(md_path) json_path.write_text(json.dumps(summary.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8") lines = [ "# Gorkhapatra Language Page Candidate Pack", "", f"- manifest: `{summary.manifest_path}`", f"- samples: {summary.sample_count}", f"- copied page candidates: {summary.copied_page_count}", "", "## Reference Status", "", ] lines.extend(f"- `{key}`: {value}" for key, value in summary.reference_status_counts.items()) if summary.reference_status_counts else lines.append("- none") lines.extend(["", "## Languages", ""]) lines.extend(f"- {language}: {count}" for language, count in summary.language_counts.items()) if summary.language_counts else lines.append("- none") lines.extend(["", "## Warnings", ""]) lines.extend(f"- {warning}" for warning in summary.warnings) if summary.warnings else lines.append("- none") lines.extend( [ "", "## Claim Status", "", "All rows are `claim_evidence_eligible=false` and `pending_manual_label` until page-language disambiguation and reference transcription are complete.", ] ) md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") def _write_gorkhapatra_language_page_review( summary: GorkhapatraLanguagePageReviewSummary, rows: list[dict[str, Any]], ) -> None: json_path = Path(summary.review_json_path) csv_path = Path(summary.review_csv_path) md_path = Path(summary.review_md_path) payload = { "summary": summary.to_dict(), "rows": rows, "review_decision_values": [ "accept_language_page", "reject_wrong_language", "reject_not_language_page", "duplicate_candidate", "needs_additional_review", ], } json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") fieldnames = [ "sample_id", "candidate_language", "date_label", "date_key", "pdf_page_index", "pdf_page_number", "image_path", "reference_path", "article_url", "source_image_url", "source_url", "pdf_path", "pdf_sha256", "language_page_marker", "alignment_status", "page_disambiguation_status", "reference_status", "claim_evidence_eligible", "review_decision", "reviewer", "reviewed_at", "notes", ] with csv_path.open("w", encoding="utf-8", newline="") as handle: writer = csv.DictWriter(handle, fieldnames=fieldnames) writer.writeheader() for row in rows: writer.writerow({field: row.get(field) for field in fieldnames}) lines = [ "# Gorkhapatra Language Page Review", "", f"- manifest: `{summary.manifest_path}`", f"- samples: {summary.sample_count}", f"- pending review: {summary.pending_review_count}", f"- claim eligible: {summary.claim_evidence_eligible_count}", f"- CSV: `{summary.review_csv_path}`", "", "## Review Rule", "", "Rows remain non-claim-ready until a reviewer confirms the language-page match and creates a verified reference transcript/layout.", "", "## Languages", "", ] lines.extend(f"- {language}: {count}" for language, count in summary.language_counts.items()) if summary.language_counts else lines.append("- none") lines.extend(["", "## Same Page Candidate Counts", ""]) lines.extend(f"- `{key}`: {count}" for key, count in summary.page_candidate_counts.items()) if summary.page_candidate_counts else lines.append("- none") if summary.warnings: lines.extend(["", "## Warnings", ""]) lines.extend(f"- {warning}" for warning in summary.warnings) lines.extend(["", "## Rows", ""]) for row in rows[:100]: lines.append( f"- `{row['sample_id']}` {row.get('candidate_language') or 'unknown'} " f"date={row.get('date_key') or 'unknown'} page={row.get('pdf_page_number') or 'unknown'} " f"status={row.get('page_disambiguation_status') or 'unknown'}" ) md_path.write_text("\n".join(lines) + "\n", encoding="utf-8") def _clean_optional_str(value: Any) -> str | None: if not isinstance(value, str): return None cleaned = value.strip() return cleaned or None
"""Source audits for real-world validation data."""
from __future__ import annotations
import csv
import json
import re
import shutil
from dataclasses import dataclass, field
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, unquote, urljoin, urlparse
from urllib.request import Request, urlopen
from .errors import DataValidationError
from .manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest
DEFAULT_GORKHAPATRA_CATEGORY_URLS = (
"https://gorkhapatraonline.com/categories/new-nepal",
"https://gorkhapatraonline.com/categories/new-nepal?page=2",
)
DEFAULT_GORKHAPATRA_EPAPER_URL = (
"https://epaper.gorkhapatraonline.com/epaper-list?"
"pdfdaterange1=2026-06-09&pdfdaterange2=2026-06-16&slug=gorkhapatra"
)
GORKHAPATRA_ONLINE_BASE = "https://gorkhapatraonline.com"
GORKHAPATRA_EPAPER_BASE = "https://epaper.gorkhapatraonline.com"
DEFAULT_NAYA_NEPAL_PUBLICATION_URL = "https://gorkhapatraonline.com/publications/naya-nepal"
LANGUAGE_PAGE_MARKERS = ("भाषा पृष्ठ", "efiff k[i7", "bhasa pristha", "bhasa prishtha")
NEPALI_MONTHS = {
"बैशाख": 1,
"वैशाख": 1,
"जेठ": 2,
"असार": 3,
"साउन": 4,
"भदौ": 5,
"भाद्र": 5,
"असोज": 6,
"कार्तिक": 7,
"मंसिर": 8,
"मङ्सिर": 8,
"पुस": 9,
"पुष": 9,
"माघ": 10,
"फागुन": 11,
"चैत": 12,
}
@dataclass(slots=True)
class DownloadedArtifact:
url: str
path: str
sha256: str
size_bytes: int
kind: str
warnings: list[str] = field(default_factory=list)
pdf_audit: dict[str, Any] | None = None
def to_dict(self) -> dict[str, Any]:
payload = {
"url": self.url,
"path": self.path,
"sha256": self.sha256,
"size_bytes": self.size_bytes,
"kind": self.kind,
"warnings": self.warnings,
}
if self.pdf_audit is not None:
payload["pdf_audit"] = self.pdf_audit
return payload
@dataclass(slots=True)
class GorkhapatraArticleAsset:
article_url: str | None
image_url: str | None
title: str
language: str | None
source_page: str
downloaded_image: DownloadedArtifact | None = None
def to_dict(self) -> dict[str, Any]:
return {
"article_url": self.article_url,
"image_url": self.image_url,
"title": self.title,
"language": self.language,
"source_page": self.source_page,
"downloaded_image": self.downloaded_image.to_dict() if self.downloaded_image else None,
}
@dataclass(slots=True)
class GorkhapatraEpaperAsset:
viewer_url: str