Add 195 files (runs/hf-recognizer-mixed-v2-full/finalized/package/artifacts/model/model.safetensors ...)
1ef1de7 verified Download src/ocrtech/source_audit.py from prosperious/ocr-tech-k2: direct link, hf CLI and curl.
- Browser
- Download file 67.3 kB
-
https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/src/ocrtech/source_audit.py
- Command line
-
hf download hf://prosperious/ocr-tech-k2/src/ocrtech/source_audit.py
-
curl -L -o source_audit.py https://huggingface.co/prosperious/ocr-tech-k2/resolve/main/src/ocrtech/source_audit.py
67.3 kB
"""Source audits for real-world validation data."""
from __future__ import annotations
import csv
import json
import re
import shutil
from dataclasses import dataclass, field
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, unquote, urljoin, urlparse
from urllib.request import Request, urlopen
from .errors import DataValidationError
from .manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest
DEFAULT_GORKHAPATRA_CATEGORY_URLS = (
"https://gorkhapatraonline.com/categories/new-nepal",
"https://gorkhapatraonline.com/categories/new-nepal?page=2",
)
DEFAULT_GORKHAPATRA_EPAPER_URL = (
"https://epaper.gorkhapatraonline.com/epaper-list?"
"pdfdaterange1=2026-06-09&pdfdaterange2=2026-06-16&slug=gorkhapatra"
)
GORKHAPATRA_ONLINE_BASE = "https://gorkhapatraonline.com"
GORKHAPATRA_EPAPER_BASE = "https://epaper.gorkhapatraonline.com"
DEFAULT_NAYA_NEPAL_PUBLICATION_URL = "https://gorkhapatraonline.com/publications/naya-nepal"
LANGUAGE_PAGE_MARKERS = ("भाषा पृष्ठ", "efiff k[i7", "bhasa pristha", "bhasa prishtha")
NEPALI_MONTHS = {
"बैशाख": 1,
"वैशाख": 1,
"जेठ": 2,
"असार": 3,
"साउन": 4,
"भदौ": 5,
"भाद्र": 5,
"असोज": 6,
"कार्तिक": 7,
"मंसिर": 8,
"मङ्सिर": 8,
"पुस": 9,
"पुष": 9,
"माघ": 10,
"फागुन": 11,
"चैत": 12,
}
@dataclass(slots=True)
class DownloadedArtifact:
url: str
path: str
sha256: str
size_bytes: int
kind: str
warnings: list[str] = field(default_factory=list)
pdf_audit: dict[str, Any] | None = None
def to_dict(self) -> dict[str, Any]:
payload = {
"url": self.url,
"path": self.path,
"sha256": self.sha256,
"size_bytes": self.size_bytes,
"kind": self.kind,
"warnings": self.warnings,
}
if self.pdf_audit is not None:
payload["pdf_audit"] = self.pdf_audit
return payload
@dataclass(slots=True)
class GorkhapatraArticleAsset:
article_url: str | None
image_url: str | None
title: str
language: str | None
source_page: str
downloaded_image: DownloadedArtifact | None = None
def to_dict(self) -> dict[str, Any]:
return {
"article_url": self.article_url,
"image_url": self.image_url,
"title": self.title,
"language": self.language,
"source_page": self.source_page,
"downloaded_image": self.downloaded_image.to_dict() if self.downloaded_image else None,
}
@dataclass(slots=True)
class GorkhapatraEpaperAsset:
viewer_url: str
direct_pdf_url: str
source_page: str
date_label: str | None = None
thumbnail_url: str | None = None
downloaded_pdf: DownloadedArtifact | None = None
def to_dict(self) -> dict[str, Any]:
return {
"viewer_url": self.viewer_url,
"direct_pdf_url": self.direct_pdf_url,
"source_page": self.source_page,
"date_label": self.date_label,
"thumbnail_url": self.thumbnail_url,
"downloaded_pdf": self.downloaded_pdf.to_dict() if self.downloaded_pdf else None,
}
@dataclass(slots=True)
class GorkhapatraSourceAudit:
category_sources: list[str]
epaper_sources: list[str]
article_count: int
epaper_count: int
language_counts: dict[str, int]
downloaded_count: int
articles: list[GorkhapatraArticleAsset]
epapers: list[GorkhapatraEpaperAsset]
warnings: list[str] = field(default_factory=list)
summary_json_path: str | None = None
summary_md_path: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"category_sources": self.category_sources,
"epaper_sources": self.epaper_sources,
"article_count": self.article_count,
"epaper_count": self.epaper_count,
"language_counts": self.language_counts,
"downloaded_count": self.downloaded_count,
"warnings": self.warnings,
"articles": [article.to_dict() for article in self.articles],
"epapers": [epaper.to_dict() for epaper in self.epapers],
"summary_json_path": self.summary_json_path,
"summary_md_path": self.summary_md_path,
}
@dataclass(slots=True)
class GorkhapatraPackSummary:
manifest_path: str
sample_count: int
article_image_count: int
epaper_page_count: int
language_counts: dict[str, int]
reference_status_counts: dict[str, int]
warnings: list[str] = field(default_factory=list)
summary_json_path: str | None = None
summary_md_path: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"manifest_path": self.manifest_path,
"sample_count": self.sample_count,
"article_image_count": self.article_image_count,
"epaper_page_count": self.epaper_page_count,
"language_counts": self.language_counts,
"reference_status_counts": self.reference_status_counts,
"warnings": self.warnings,
"summary_json_path": self.summary_json_path,
"summary_md_path": self.summary_md_path,
}
@dataclass(slots=True)
class NayaNepalPublicationItem:
article_url: str | None
image_url: str | None
title: str
language: str | None
date_label: str | None
date_key: str | None
source_page: str
def to_dict(self) -> dict[str, Any]:
return {
"article_url": self.article_url,
"image_url": self.image_url,
"title": self.title,
"language": self.language,
"date_label": self.date_label,
"date_key": self.date_key,
"source_page": self.source_page,
}
@dataclass(slots=True)
class GorkhapatraLanguagePageHit:
page_index: int
marker: str
text_chars: int
text_preview: str
rendered_page_path: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"page_index": self.page_index,
"marker": self.marker,
"text_chars": self.text_chars,
"text_preview": self.text_preview,
"rendered_page_path": self.rendered_page_path,
}
@dataclass(slots=True)
class GorkhapatraLanguagePageAlignment:
publication_item: NayaNepalPublicationItem
epaper: GorkhapatraEpaperAsset | None
pdf_path: str | None
pdf_sha256: str | None
page_hits: list[GorkhapatraLanguagePageHit]
alignment_status: str
claim_evidence_eligible: bool
warnings: list[str] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"publication_item": self.publication_item.to_dict(),
"epaper": self.epaper.to_dict() if self.epaper else None,
"pdf_path": self.pdf_path,
"pdf_sha256": self.pdf_sha256,
"page_hits": [hit.to_dict() for hit in self.page_hits],
"alignment_status": self.alignment_status,
"claim_evidence_eligible": self.claim_evidence_eligible,
"warnings": self.warnings,
}
@dataclass(slots=True)
class GorkhapatraLanguagePageAudit:
publication_sources: list[str]
epaper_sources: list[str]
publication_item_count: int
aligned_count: int
language_page_hit_count: int
status_counts: dict[str, int]
language_counts: dict[str, int]
alignments: list[GorkhapatraLanguagePageAlignment]
warnings: list[str] = field(default_factory=list)
summary_json_path: str | None = None
summary_md_path: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"publication_sources": self.publication_sources,
"epaper_sources": self.epaper_sources,
"publication_item_count": self.publication_item_count,
"aligned_count": self.aligned_count,
"language_page_hit_count": self.language_page_hit_count,
"status_counts": self.status_counts,
"language_counts": self.language_counts,
"warnings": self.warnings,
"summary_json_path": self.summary_json_path,
"summary_md_path": self.summary_md_path,
"alignments": [alignment.to_dict() for alignment in self.alignments],
}
@dataclass(slots=True)
class GorkhapatraLanguagePagePackSummary:
manifest_path: str
sample_count: int
copied_page_count: int
language_counts: dict[str, int]
reference_status_counts: dict[str, int]
warnings: list[str] = field(default_factory=list)
summary_json_path: str | None = None
summary_md_path: str | None = None
def to_dict(self) -> dict[str, Any]:
return {
"manifest_path": self.manifest_path,
"sample_count": self.sample_count,
"copied_page_count": self.copied_page_count,
"language_counts": self.language_counts,
"reference_status_counts": self.reference_status_counts,
"warnings": self.warnings,
"summary_json_path": self.summary_json_path,
"summary_md_path": self.summary_md_path,
}
@dataclass(slots=True)
class GorkhapatraLanguagePageReviewSummary:
manifest_path: str
review_json_path: str
review_csv_path: str
review_md_path: str
sample_count: int
pending_review_count: int
claim_evidence_eligible_count: int
language_counts: dict[str, int]
page_candidate_counts: dict[str, int]
warnings: list[str] = field(default_factory=list)
def to_dict(self) -> dict[str, Any]:
return {
"manifest_path": self.manifest_path,
"review_json_path": self.review_json_path,
"review_csv_path": self.review_csv_path,
"review_md_path": self.review_md_path,
"sample_count": self.sample_count,
"pending_review_count": self.pending_review_count,
"claim_evidence_eligible_count": self.claim_evidence_eligible_count,
"language_counts": self.language_counts,
"page_candidate_counts": self.page_candidate_counts,
"warnings": self.warnings,
}
class _GorkhapatraCategoryParser(HTMLParser):
def __init__(self, *, base_url: str, source_page: str) -> None:
super().__init__(convert_charrefs=True)
self.base_url = base_url
self.source_page = source_page
self.assets: list[GorkhapatraArticleAsset] = []
self._href_stack: list[str | None] = []
self._link_text_stack: list[list[str]] = []
self._title_links: dict[str, str] = {}
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
attr = dict(attrs)
if tag == "a":
href = attr.get("href")
self._href_stack.append(urljoin(self.base_url, href) if href else None)
self._link_text_stack.append([])
return
if tag != "img":
return
alt = (attr.get("alt") or "").strip()
if "भाषा" not in alt:
return
src = attr.get("src")
article_url = self._nearest_article_url()
self.assets.append(
GorkhapatraArticleAsset(
article_url=article_url,
image_url=urljoin(self.base_url, src) if src else None,
title=alt,
language=_extract_language_label(alt),
source_page=self.source_page,
)
)
def handle_endtag(self, tag: str) -> None:
if tag == "a" and self._href_stack:
href = self._href_stack.pop()
text_chunks = self._link_text_stack.pop() if self._link_text_stack else []
text = " ".join(" ".join(text_chunks).split())
if href and re.search(r"/news/\d+", href) and "भाषा" in text:
self._title_links[text] = href
def handle_data(self, data: str) -> None:
if self._link_text_stack:
self._link_text_stack[-1].append(data)
def _nearest_article_url(self) -> str | None:
for href in reversed(self._href_stack):
if href and re.search(r"/news/\d+", href):
return href
return None
def finalize_assets(self) -> list[GorkhapatraArticleAsset]:
for asset in self.assets:
if asset.article_url is None:
asset.article_url = self._title_links.get(" ".join(asset.title.split()))
return self.assets
class _GorkhapatraEpaperParser(HTMLParser):
def __init__(self, *, base_url: str, source_page: str) -> None:
super().__init__(convert_charrefs=True)
self.base_url = base_url
self.source_page = source_page
self.assets: list[GorkhapatraEpaperAsset] = []
self._current_viewer: str | None = None
self._last_thumbnail: str | None = None
self._capture_date = False
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
attr = dict(attrs)
if tag == "a":
href = attr.get("href")
if href and "/pdf?file=" in href:
self._current_viewer = urljoin(self.base_url, href)
return
if tag == "img":
src = attr.get("src")
if src and "/uploads/image/" in src:
self._last_thumbnail = urljoin(self.base_url, src)
return
if tag == "span" and "date" in (attr.get("class") or ""):
self._capture_date = True
def handle_data(self, data: str) -> None:
if not self._capture_date or not self._current_viewer:
return
date_label = " ".join(data.split())
direct_pdf = _direct_epaper_pdf_url(self._current_viewer)
self.assets.append(
GorkhapatraEpaperAsset(
viewer_url=self._current_viewer,
direct_pdf_url=direct_pdf,
source_page=self.source_page,
date_label=date_label or None,
thumbnail_url=self._last_thumbnail,
)
)
self._current_viewer = None
self._last_thumbnail = None
self._capture_date = False
def handle_endtag(self, tag: str) -> None:
if tag == "span":
self._capture_date = False
class _NayaNepalPublicationParser(HTMLParser):
def __init__(self, *, base_url: str, source_page: str) -> None:
super().__init__(convert_charrefs=True)
self.base_url = base_url
self.source_page = source_page
self.items: list[NayaNepalPublicationItem] = []
self._href_stack: list[str | None] = []
self._current_index: int | None = None
self._capture_title = False
def handle_starttag(self, tag: str, attrs: list[tuple[str, str | None]]) -> None:
attr = dict(attrs)
if tag == "a":
href = attr.get("href")
self._href_stack.append(urljoin(self.base_url, href) if href else None)
return
if tag == "img":
alt = (attr.get("alt") or "").strip()
if "भाषा" not in alt:
return
src = attr.get("src")
self.items.append(
NayaNepalPublicationItem(
article_url=self._nearest_article_url(),
image_url=urljoin(self.base_url, src) if src else None,
title=alt,
language=_extract_language_label(alt),
date_label=None,
date_key=None,
source_page=self.source_page,
)
)
self._current_index = len(self.items) - 1
return
if tag in {"h2", "h3"} and self._current_index is not None:
self._capture_title = True
def handle_endtag(self, tag: str) -> None:
if tag == "a" and self._href_stack:
self._href_stack.pop()
if tag in {"h2", "h3"}:
self._capture_title = False
def handle_data(self, data: str) -> None:
if self._current_index is None:
return
text = " ".join(data.split())
if not text:
return
item = self.items[self._current_index]
date_key = _nepali_date_key(text)
if date_key and item.date_key is None:
item.date_label = _clean_date_label(text)
item.date_key = date_key
if self._capture_title and "भाषा" in text:
item.title = text
item.language = _extract_language_label(text)
def _nearest_article_url(self) -> str | None:
for href in reversed(self._href_stack):
if href and re.search(r"/news/\d+", href):
return href
return None
def audit_gorkhapatra_source(
output_dir: str | Path,
*,
category_urls: list[str] | None = None,
epaper_urls: list[str] | None = None,
category_html_paths: list[str | Path] | None = None,
epaper_html_paths: list[str | Path] | None = None,
download_assets: bool = False,
max_article_images: int = 0,
max_epaper_pdfs: int = 0,
timeout_seconds: float = 30.0,
) -> GorkhapatraSourceAudit:
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
warnings: list[str] = []
category_sources = list(category_urls or ())
epaper_sources = list(epaper_urls or ())
category_inputs = _read_html_inputs(category_sources, category_html_paths or [], timeout_seconds=timeout_seconds, output_dir=out, prefix="category")
epaper_inputs = _read_html_inputs(epaper_sources, epaper_html_paths or [], timeout_seconds=timeout_seconds, output_dir=out, prefix="epaper")
articles: list[GorkhapatraArticleAsset] = []
for source_name, html_text in category_inputs:
parser = _GorkhapatraCategoryParser(base_url=GORKHAPATRA_ONLINE_BASE, source_page=source_name)
parser.feed(html_text)
articles.extend(parser.finalize_assets())
epapers: list[GorkhapatraEpaperAsset] = []
for source_name, html_text in epaper_inputs:
parser = _GorkhapatraEpaperParser(base_url=GORKHAPATRA_EPAPER_BASE, source_page=source_name)
parser.feed(html_text)
epapers.extend(parser.assets)
articles = _dedupe_articles(articles)
epapers = _dedupe_epapers(epapers)
if download_assets:
_download_article_images(articles, out, max_count=max_article_images, timeout_seconds=timeout_seconds, warnings=warnings)
_download_epaper_pdfs(epapers, out, max_count=max_epaper_pdfs, timeout_seconds=timeout_seconds, warnings=warnings)
language_counts: dict[str, int] = {}
for article in articles:
if article.language:
language_counts[article.language] = language_counts.get(article.language, 0) + 1
downloaded_count = sum(1 for article in articles if article.downloaded_image) + sum(1 for epaper in epapers if epaper.downloaded_pdf)
audit = GorkhapatraSourceAudit(
category_sources=[source for source, _ in category_inputs],
epaper_sources=[source for source, _ in epaper_inputs],
article_count=len(articles),
epaper_count=len(epapers),
language_counts=dict(sorted(language_counts.items())),
downloaded_count=downloaded_count,
articles=articles,
epapers=epapers,
warnings=warnings,
)
_write_gorkhapatra_audit(audit, out)
return audit
def prepare_gorkhapatra_pack(
audit_json_path: str | Path,
output_dir: str | Path,
*,
include_article_images: bool = True,
include_epaper_pages: bool = True,
max_article_images: int = 0,
max_epaper_pdfs: int = 0,
max_pages_per_pdf: int = 2,
split: str = "eval",
dataset_name: str = "gorkhapatra-naya-nepal",
) -> GorkhapatraPackSummary:
audit_path = Path(audit_json_path)
if not audit_path.is_file():
raise DataValidationError(f"Gorkhapatra source audit JSON does not exist: {audit_path}")
if max_pages_per_pdf < 1:
raise DataValidationError("max_pages_per_pdf must be at least 1")
payload = json.loads(audit_path.read_text(encoding="utf-8"))
out = Path(output_dir)
inputs_dir = out / "inputs"
refs_dir = out / "references"
manifests_dir = out / "manifests"
inputs_dir.mkdir(parents=True, exist_ok=True)
refs_dir.mkdir(parents=True, exist_ok=True)
manifests_dir.mkdir(parents=True, exist_ok=True)
entries: list[ManifestEntry] = []
warnings: list[str] = []
article_count = 0
epaper_page_count = 0
if include_article_images:
for article_index, article in enumerate(payload.get("articles") or [], start=1):
if max_article_images and article_count >= max_article_images:
break
downloaded = article.get("downloaded_image") if isinstance(article, dict) else None
if not isinstance(downloaded, dict) or not downloaded.get("path"):
continue
source_path = _resolve_audit_artifact_path(str(downloaded["path"]), audit_path)
if not source_path.is_file():
warnings.append(f"missing downloaded article image for article {article_index}: {source_path}")
continue
sample_id = f"gorkhapatra-article-{article_index:04d}"
target_path = inputs_dir / f"{sample_id}{source_path.suffix.lower() or '.jpg'}"
shutil.copy2(source_path, target_path)
reference_path = refs_dir / f"{sample_id}.ref.json"
language = _clean_optional_str(article.get("language"))
source_url = _clean_optional_str(article.get("article_url")) or _clean_optional_str(article.get("image_url"))
_write_pending_reference(
reference_path,
sample_id=sample_id,
source_url=source_url,
language=language,
source_kind="gorkhapatra_article_image",
)
image_hash = sha256_file(target_path)
entries.append(
ManifestEntry(
sample_id=sample_id,
dataset=dataset_name,
split=split,
image_path=str(target_path),
text="",
sha256=image_hash,
metadata={
"benchmark_name": "gorkhapatra-naya-nepal",
"source": "gorkhapatra",
"source_kind": "article_image",
"source_url": source_url,
"source_image_url": article.get("image_url"),
"language": language,
"script": "unknown",
"input_format": "image",
"document_type": "article_image",
"reference_path": str(reference_path),
"reference_status": "pending_manual_label",
"claim_evidence_eligible": False,
"slices": _gorkhapatra_slices(language, "article_image"),
"text_sha256": sha256_text(""),
"sample_sha256": sha256_text(f"{image_hash}\n"),
},
)
)
article_count += 1
if include_epaper_pages:
epapers = payload.get("epapers") or []
rendered_pdf_count = 0
for epaper_index, epaper in enumerate(epapers, start=1):
if max_epaper_pdfs and rendered_pdf_count >= max_epaper_pdfs:
break
downloaded = epaper.get("downloaded_pdf") if isinstance(epaper, dict) else None
if not isinstance(downloaded, dict) or not downloaded.get("path"):
continue
source_path = _resolve_audit_artifact_path(str(downloaded["path"]), audit_path)
if not source_path.is_file():
warnings.append(f"missing downloaded epaper PDF for epaper {epaper_index}: {source_path}")
continue
try:
rendered_pages = _render_pdf_pages(source_path, inputs_dir, epaper_index=epaper_index, max_pages=max_pages_per_pdf)
except DataValidationError as exc:
warnings.append(str(exc))
continue
rendered_pdf_count += 1
for page_index, target_path in rendered_pages:
sample_id = f"gorkhapatra-epaper-{epaper_index:04d}-page-{page_index + 1:04d}"
reference_path = refs_dir / f"{sample_id}.ref.json"
source_url = _clean_optional_str(epaper.get("direct_pdf_url")) or _clean_optional_str(epaper.get("viewer_url"))
_write_pending_reference(
reference_path,
sample_id=sample_id,
source_url=source_url,
language=None,
source_kind="gorkhapatra_epaper_page",
)
image_hash = sha256_file(target_path)
entries.append(
ManifestEntry(
sample_id=sample_id,
dataset=dataset_name,
split=split,
image_path=str(target_path),
text="",
sha256=image_hash,
metadata={
"benchmark_name": "gorkhapatra-naya-nepal",
"source": "gorkhapatra",
"source_kind": "epaper_page",
"source_url": source_url,
"viewer_url": epaper.get("viewer_url"),
"date_label": epaper.get("date_label"),
"pdf_sha256": downloaded.get("sha256"),
"pdf_page_index": page_index,
"language": None,
"script": "unknown",
"input_format": "image",
"document_type": "epaper_page",
"reference_path": str(reference_path),
"reference_status": "pending_manual_label",
"claim_evidence_eligible": False,
"slices": _gorkhapatra_slices(None, "epaper_page"),
"text_sha256": sha256_text(""),
"sample_sha256": sha256_text(f"{image_hash}\n"),
},
)
)
epaper_page_count += 1
manifest_path = manifests_dir / "gorkhapatra-pack.jsonl"
write_manifest(entries, manifest_path)
summary = _gorkhapatra_pack_summary(
manifest_path,
entries,
article_image_count=article_count,
epaper_page_count=epaper_page_count,
warnings=warnings,
)
_write_gorkhapatra_pack_summary(summary, out)
return summary
def audit_gorkhapatra_language_pages(
output_dir: str | Path,
*,
publication_urls: list[str] | None = None,
epaper_urls: list[str] | None = None,
publication_html_paths: list[str | Path] | None = None,
epaper_html_paths: list[str | Path] | None = None,
download_pdfs: bool = False,
render_pages: bool = False,
max_publication_items: int = 0,
max_epaper_pdfs: int = 0,
timeout_seconds: float = 30.0,
) -> GorkhapatraLanguagePageAudit:
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
warnings: list[str] = []
publication_sources = list(publication_urls or ())
epaper_sources = list(epaper_urls or ())
publication_inputs = _read_html_inputs(
publication_sources,
publication_html_paths or [],
timeout_seconds=timeout_seconds,
output_dir=out,
prefix="naya-nepal-publication",
)
epaper_inputs = _read_html_inputs(epaper_sources, epaper_html_paths or [], timeout_seconds=timeout_seconds, output_dir=out, prefix="epaper")
publication_items: list[NayaNepalPublicationItem] = []
for source_name, html_text in publication_inputs:
parser = _NayaNepalPublicationParser(base_url=GORKHAPATRA_ONLINE_BASE, source_page=source_name)
parser.feed(html_text)
publication_items.extend(parser.items)
publication_items = _dedupe_publication_items(publication_items)
if max_publication_items:
publication_items = publication_items[:max_publication_items]
epapers: list[GorkhapatraEpaperAsset] = []
for source_name, html_text in epaper_inputs:
parser = _GorkhapatraEpaperParser(base_url=GORKHAPATRA_EPAPER_BASE, source_page=source_name)
parser.feed(html_text)
epapers.extend(parser.assets)
epapers = _dedupe_epapers(epapers)
epapers_by_date = {_nepali_date_key(epaper.date_label or ""): epaper for epaper in epapers if _nepali_date_key(epaper.date_label or "")}
alignments: list[GorkhapatraLanguagePageAlignment] = []
downloaded_pdf_count = 0
pdf_cache: dict[str, DownloadedArtifact] = {}
for index, item in enumerate(publication_items, start=1):
item_warnings: list[str] = []
if not item.date_key:
alignments.append(_language_page_alignment(item, None, None, None, [], "missing_publication_date", item_warnings))
continue
epaper = epapers_by_date.get(item.date_key)
if epaper is None:
alignments.append(_language_page_alignment(item, None, None, None, [], "no_matching_epaper_date", item_warnings))
continue
if not download_pdfs:
alignments.append(_language_page_alignment(item, epaper, None, None, [], "matched_epaper_not_downloaded", item_warnings))
continue
if max_epaper_pdfs and downloaded_pdf_count >= max_epaper_pdfs and epaper.direct_pdf_url not in pdf_cache:
alignments.append(_language_page_alignment(item, epaper, None, None, [], "matched_epaper_not_downloaded_limit", item_warnings))
continue
try:
artifact = pdf_cache.get(epaper.direct_pdf_url)
if artifact is None:
target = out / "epapers" / f"epaper-{item.date_key}.pdf"
artifact = _download_url(epaper.direct_pdf_url, target, timeout_seconds=timeout_seconds)
artifact.pdf_audit = _audit_pdf_with_pymupdf(target)
pdf_cache[epaper.direct_pdf_url] = artifact
downloaded_pdf_count += 1
epaper.downloaded_pdf = artifact
hits = _find_language_page_hits(Path(artifact.path), out / "language-pages", render_pages=render_pages)
status = "language_page_found" if hits else "matched_epaper_no_language_page_marker"
alignments.append(_language_page_alignment(item, epaper, artifact.path, artifact.sha256, hits, status, item_warnings))
except Exception as exc: # noqa: BLE001 - recorded as provenance warning, not ignored.
warning = f"{type(exc).__name__}: {exc}"
item_warnings.append(warning)
warnings.append(f"failed to inspect {epaper.direct_pdf_url}: {warning}")
alignments.append(_language_page_alignment(item, epaper, None, None, [], "pdf_inspection_failed", item_warnings))
audit = _gorkhapatra_language_page_audit(
publication_sources=[source for source, _ in publication_inputs],
epaper_sources=[source for source, _ in epaper_inputs],
publication_items=publication_items,
alignments=alignments,
warnings=warnings,
)
_write_gorkhapatra_language_page_audit(audit, out)
return audit
def prepare_gorkhapatra_language_page_pack(
language_page_audit_json_path: str | Path,
output_dir: str | Path,
*,
split: str = "eval",
dataset_name: str = "gorkhapatra-language-pages",
max_samples: int = 0,
) -> GorkhapatraLanguagePagePackSummary:
audit_path = Path(language_page_audit_json_path)
if not audit_path.is_file():
raise DataValidationError(f"Gorkhapatra language-page audit JSON does not exist: {audit_path}")
payload = json.loads(audit_path.read_text(encoding="utf-8"))
out = Path(output_dir)
inputs_dir = out / "inputs"
refs_dir = out / "references"
manifests_dir = out / "manifests"
inputs_dir.mkdir(parents=True, exist_ok=True)
refs_dir.mkdir(parents=True, exist_ok=True)
manifests_dir.mkdir(parents=True, exist_ok=True)
entries: list[ManifestEntry] = []
warnings: list[str] = []
copied_count = 0
for alignment_index, alignment in enumerate(payload.get("alignments") or [], start=1):
if max_samples and len(entries) >= max_samples:
break
if not isinstance(alignment, dict):
continue
publication_item = alignment.get("publication_item")
if not isinstance(publication_item, dict):
warnings.append(f"alignment {alignment_index} missing publication_item")
continue
language = _clean_optional_str(publication_item.get("language"))
date_key = _clean_optional_str(publication_item.get("date_key")) or "unknown-date"
page_hits = alignment.get("page_hits")
if not isinstance(page_hits, list) or not page_hits:
warnings.append(f"alignment {alignment_index} has no rendered page hits")
continue
for hit_index, hit in enumerate(page_hits, start=1):
if max_samples and len(entries) >= max_samples:
break
if not isinstance(hit, dict):
continue
rendered_path_value = _clean_optional_str(hit.get("rendered_page_path"))
if not rendered_path_value:
warnings.append(f"alignment {alignment_index} page hit {hit_index} has no rendered_page_path")
continue
source_path = _resolve_audit_artifact_path(rendered_path_value, audit_path)
if not source_path.is_file():
warnings.append(f"alignment {alignment_index} page hit {hit_index} missing rendered page: {source_path}")
continue
page_index = int(hit.get("page_index") or 0)
sample_id = f"gorkhapatra-language-page-{alignment_index:04d}-{hit_index:02d}"
target_path = inputs_dir / f"{sample_id}{source_path.suffix.lower() or '.png'}"
shutil.copy2(source_path, target_path)
reference_path = refs_dir / f"{sample_id}.ref.json"
source_url = _alignment_source_url(alignment)
_write_pending_reference(
reference_path,
sample_id=sample_id,
source_url=source_url,
language=language,
source_kind="gorkhapatra_language_page_candidate",
)
image_hash = sha256_file(target_path)
entries.append(
ManifestEntry(
sample_id=sample_id,
dataset=dataset_name,
split=split,
image_path=str(target_path),
text="",
sha256=image_hash,
metadata={
"benchmark_name": "gorkhapatra-naya-nepal-language-pages",
"source": "gorkhapatra",
"source_kind": "language_page_candidate",
"source_url": source_url,
"article_url": publication_item.get("article_url"),
"source_image_url": publication_item.get("image_url"),
"candidate_language": language,
"language": language,
"date_label": publication_item.get("date_label"),
"date_key": date_key,
"pdf_path": alignment.get("pdf_path"),
"pdf_sha256": alignment.get("pdf_sha256"),
"pdf_page_index": page_index,
"pdf_page_number": page_index + 1,
"language_page_marker": hit.get("marker"),
"alignment_status": alignment.get("alignment_status"),
"page_disambiguation_status": "candidate_requires_manual_review",
"script": "unknown",
"input_format": "image",
"document_type": "language_page_candidate",
"reference_path": str(reference_path),
"reference_status": "pending_manual_label",
"claim_evidence_eligible": False,
"slices": _gorkhapatra_slices(language, "language_page_candidate"),
"text_sha256": sha256_text(""),
"sample_sha256": sha256_text(f"{image_hash}\n{alignment.get('pdf_sha256') or ''}\n{page_index}\n"),
},
)
)
copied_count += 1
manifest_path = manifests_dir / "gorkhapatra-language-page-pack.jsonl"
write_manifest(entries, manifest_path)
summary = _gorkhapatra_language_page_pack_summary(
manifest_path,
entries,
copied_page_count=copied_count,
warnings=warnings,
)
_write_gorkhapatra_language_page_pack_summary(summary, out)
return summary
def prepare_gorkhapatra_language_page_review(
manifest_path: str | Path,
output_dir: str | Path,
) -> GorkhapatraLanguagePageReviewSummary:
source_manifest = Path(manifest_path)
if not source_manifest.is_file():
raise DataValidationError(f"Gorkhapatra language-page pack manifest does not exist: {source_manifest}")
entries = load_manifest(source_manifest)
out = Path(output_dir)
out.mkdir(parents=True, exist_ok=True)
rows: list[dict[str, Any]] = []
warnings: list[str] = []
language_counts: dict[str, int] = {}
page_candidate_counts: dict[str, int] = {}
for entry in entries:
metadata = entry.metadata or {}
source_kind = metadata.get("source_kind")
if source_kind != "language_page_candidate":
warnings.append(f"{entry.sample_id} source_kind is not language_page_candidate: {source_kind}")
candidate_language = _clean_optional_str(metadata.get("candidate_language")) or _clean_optional_str(metadata.get("language"))
if candidate_language:
language_counts[candidate_language] = language_counts.get(candidate_language, 0) + 1
date_key = _clean_optional_str(metadata.get("date_key")) or "unknown-date"
page_number = metadata.get("pdf_page_number")
page_key = f"{date_key}:page-{page_number if page_number is not None else 'unknown'}"
page_candidate_counts[page_key] = page_candidate_counts.get(page_key, 0) + 1
rows.append(
{
"sample_id": entry.sample_id,
"candidate_language": candidate_language,
"date_label": metadata.get("date_label"),
"date_key": metadata.get("date_key"),
"pdf_page_index": metadata.get("pdf_page_index"),
"pdf_page_number": metadata.get("pdf_page_number"),
"image_path": entry.image_path,
"reference_path": metadata.get("reference_path"),
"article_url": metadata.get("article_url"),
"source_image_url": metadata.get("source_image_url"),
"source_url": metadata.get("source_url"),
"pdf_path": metadata.get("pdf_path"),
"pdf_sha256": metadata.get("pdf_sha256"),
"language_page_marker": metadata.get("language_page_marker"),
"alignment_status": metadata.get("alignment_status"),
"page_disambiguation_status": metadata.get("page_disambiguation_status"),
"reference_status": metadata.get("reference_status"),
"claim_evidence_eligible": bool(metadata.get("claim_evidence_eligible")),
"review_decision": "",
"reviewer": "",
"reviewed_at": "",
"notes": "",
}
)
json_path = out / "gorkhapatra-language-page-review.json"
csv_path = out / "gorkhapatra-language-page-review.csv"
md_path = out / "gorkhapatra-language-page-review.md"
summary = GorkhapatraLanguagePageReviewSummary(
manifest_path=str(source_manifest),
review_json_path=str(json_path),
review_csv_path=str(csv_path),
review_md_path=str(md_path),
sample_count=len(rows),
pending_review_count=sum(1 for row in rows if row["page_disambiguation_status"] == "candidate_requires_manual_review"),
claim_evidence_eligible_count=sum(1 for row in rows if row["claim_evidence_eligible"]),
language_counts=dict(sorted(language_counts.items())),
page_candidate_counts=dict(sorted(page_candidate_counts.items())),
warnings=warnings,
)
_write_gorkhapatra_language_page_review(summary, rows)
return summary
def _read_html_inputs(
urls: list[str],
paths: list[str | Path],
*,
timeout_seconds: float,
output_dir: Path,
prefix: str,
) -> list[tuple[str, str]]:
inputs: list[tuple[str, str]] = []
raw_dir = output_dir / "raw"
raw_dir.mkdir(parents=True, exist_ok=True)
for index, url in enumerate(urls, start=1):
text = _fetch_text(url, timeout_seconds=timeout_seconds)
raw_path = raw_dir / f"{prefix}-url-{index}.html"
raw_path.write_text(text, encoding="utf-8")
inputs.append((url, text))
for path in paths:
html_path = Path(path)
if not html_path.is_file():
raise DataValidationError(f"{prefix} HTML path does not exist: {html_path}")
inputs.append((str(html_path), html_path.read_text(encoding="utf-8")))
return inputs
def _fetch_text(url: str, *, timeout_seconds: float) -> str:
request = Request(url, headers={"User-Agent": "ocrtech-source-audit/0.1"})
with urlopen(request, timeout=timeout_seconds) as response:
data = response.read()
return data.decode("utf-8", errors="replace")
def _download_url(url: str, path: Path, *, timeout_seconds: float) -> DownloadedArtifact:
path.parent.mkdir(parents=True, exist_ok=True)
request = Request(url, headers={"User-Agent": "ocrtech-source-audit/0.1"})
with urlopen(request, timeout=timeout_seconds) as response:
path.write_bytes(response.read())
return DownloadedArtifact(url=url, path=str(path), sha256=sha256_file(path), size_bytes=path.stat().st_size, kind=_artifact_kind(path))
def _download_article_images(
articles: list[GorkhapatraArticleAsset],
output_dir: Path,
*,
max_count: int,
timeout_seconds: float,
warnings: list[str],
) -> None:
count = 0
for index, article in enumerate(articles, start=1):
if max_count and count >= max_count:
return
if not article.image_url:
continue
suffix = _url_suffix(article.image_url, default=".jpg")
target = output_dir / "article-images" / f"article-{index:04d}{suffix}"
try:
article.downloaded_image = _download_url(article.image_url, target, timeout_seconds=timeout_seconds)
count += 1
except Exception as exc: # noqa: BLE001 - wrapped as provenance warning, not ignored.
warnings.append(f"failed to download article image {article.image_url}: {type(exc).__name__}: {exc}")
def _download_epaper_pdfs(
epapers: list[GorkhapatraEpaperAsset],
output_dir: Path,
*,
max_count: int,
timeout_seconds: float,
warnings: list[str],
) -> None:
count = 0
for index, epaper in enumerate(epapers, start=1):
if max_count and count >= max_count:
return
target = output_dir / "epapers" / f"epaper-{index:04d}.pdf"
try:
artifact = _download_url(epaper.direct_pdf_url, target, timeout_seconds=timeout_seconds)
artifact.pdf_audit = _audit_pdf_with_pymupdf(target)
epaper.downloaded_pdf = artifact
count += 1
except Exception as exc: # noqa: BLE001 - wrapped as provenance warning, not ignored.
warnings.append(f"failed to download epaper PDF {epaper.direct_pdf_url}: {type(exc).__name__}: {exc}")
def _audit_pdf_with_pymupdf(path: Path) -> dict[str, Any]:
try:
import fitz # type: ignore[import-not-found]
except Exception as exc: # noqa: BLE001 - optional dependency.
return {"available": False, "error": f"{type(exc).__name__}: {exc}"}
with fitz.open(path) as document:
pages: list[dict[str, Any]] = []
total_text_chars = 0
for index, page in enumerate(document):
text = page.get_text("text") or ""
total_text_chars += len(text)
if index < 5:
pages.append(
{
"page_index": index,
"text_chars": len(text),
"text_preview": text[:300],
"block_count": len(page.get_text("blocks") or []),
"image_count": len(page.get_images(full=True) or []),
}
)
return {
"available": True,
"page_count": document.page_count,
"total_text_chars": total_text_chars,
"first_pages": pages,
"text_layer_warning": _pdf_text_layer_warning(pages),
}
def _pdf_text_layer_warning(pages: list[dict[str, Any]]) -> str | None:
preview = "\n".join(str(page.get("text_preview") or "") for page in pages)
if not preview.strip():
return "no_text_extracted"
devanagari = sum(1 for char in preview if "\u0900" <= char <= "\u097f")
ascii_like = sum(1 for char in preview if char.isascii() and char.isalnum())
if ascii_like > devanagari * 5 and devanagari < 20:
return "extracted_text_may_be_legacy_encoded_or_mojibake"
return None
def _find_language_page_hits(pdf_path: Path, output_dir: Path, *, render_pages: bool) -> list[GorkhapatraLanguagePageHit]:
try:
import fitz # type: ignore[import-not-found]
except Exception as exc: # noqa: BLE001 - optional dependency.
raise DataValidationError(f"PDF page marker search requires PyMuPDF: {type(exc).__name__}: {exc}") from exc
hits: list[GorkhapatraLanguagePageHit] = []
if render_pages:
output_dir.mkdir(parents=True, exist_ok=True)
with fitz.open(pdf_path) as document:
for page_index, page in enumerate(document):
text = page.get_text("text") or ""
marker = _language_page_marker(text)
if marker is None:
continue
rendered_path: str | None = None
if render_pages:
target = output_dir / f"{pdf_path.stem}-page-{page_index + 1:04d}.png"
pixmap = page.get_pixmap(matrix=fitz.Matrix(2.0, 2.0), alpha=False)
pixmap.save(target)
rendered_path = str(target)
hits.append(
GorkhapatraLanguagePageHit(
page_index=page_index,
marker=marker,
text_chars=len(text),
text_preview=" ".join(text.split())[:500],
rendered_page_path=rendered_path,
)
)
return hits
def _language_page_marker(text: str) -> str | None:
normalized = " ".join(text.split()).lower()
for marker in LANGUAGE_PAGE_MARKERS:
if marker.lower() in normalized:
return marker
if "भाषा" in normalized and "पृष्ठ" in normalized:
return "भाषा+पृष्ठ"
return None
def _write_gorkhapatra_audit(audit: GorkhapatraSourceAudit, output_dir: Path) -> None:
json_path = output_dir / "gorkhapatra-source-audit.json"
md_path = output_dir / "gorkhapatra-source-audit.md"
audit.summary_json_path = str(json_path)
audit.summary_md_path = str(md_path)
json_path.write_text(json.dumps(audit.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
lines = [
"# Gorkhapatra Source Audit",
"",
f"- article assets: {audit.article_count}",
f"- epaper assets: {audit.epaper_count}",
f"- downloaded artifacts: {audit.downloaded_count}",
f"- warnings: {len(audit.warnings)}",
"",
"## Languages",
"",
]
if audit.language_counts:
lines.extend(f"- {language}: {count}" for language, count in audit.language_counts.items())
else:
lines.append("- none detected")
if audit.warnings:
lines.extend(["", "## Warnings", ""])
lines.extend(f"- {warning}" for warning in audit.warnings)
lines.extend(["", "## Sample Article Assets", ""])
for article in audit.articles[:20]:
lines.append(f"- {article.language or 'unknown'}: {article.title} ({article.article_url or 'no article URL'})")
lines.extend(["", "## Sample Epaper Assets", ""])
for epaper in audit.epapers[:20]:
lines.append(f"- {epaper.date_label or 'unknown date'}: {epaper.direct_pdf_url}")
md_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _write_gorkhapatra_language_page_audit(audit: GorkhapatraLanguagePageAudit, output_dir: Path) -> None:
json_path = output_dir / "gorkhapatra-language-pages.json"
md_path = output_dir / "gorkhapatra-language-pages.md"
audit.summary_json_path = str(json_path)
audit.summary_md_path = str(md_path)
json_path.write_text(json.dumps(audit.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
lines = [
"# Gorkhapatra Language Page Alignment",
"",
f"- publication items: {audit.publication_item_count}",
f"- aligned to epaper date: {audit.aligned_count}",
f"- language page hits: {audit.language_page_hit_count}",
f"- warnings: {len(audit.warnings)}",
"",
"## Status",
"",
]
lines.extend(f"- `{status}`: {count}" for status, count in audit.status_counts.items()) if audit.status_counts else lines.append("- none")
lines.extend(["", "## Languages", ""])
lines.extend(f"- {language}: {count}" for language, count in audit.language_counts.items()) if audit.language_counts else lines.append("- none")
lines.extend(["", "## Alignments", ""])
for alignment in audit.alignments[:50]:
item = alignment.publication_item
page_list = ", ".join(str(hit.page_index + 1) for hit in alignment.page_hits) or "none"
lines.append(
f"- {item.date_label or 'unknown date'} / {item.language or 'unknown'}: "
f"`{alignment.alignment_status}` pages={page_list} title={item.title}"
)
if audit.warnings:
lines.extend(["", "## Warnings", ""])
lines.extend(f"- {warning}" for warning in audit.warnings)
md_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _direct_epaper_pdf_url(viewer_url: str) -> str:
parsed = urlparse(viewer_url)
values = parse_qs(parsed.query).get("file")
if not values:
raise DataValidationError(f"epaper viewer URL does not include file parameter: {viewer_url}")
return urljoin(GORKHAPATRA_EPAPER_BASE, unquote(values[0]))
def _extract_language_label(title: str) -> str | None:
match = re.search(r"प्रकाशित\s+(.+?)\s+भाषा", title)
if not match:
return None
return " ".join(match.group(1).split())
def _nepali_date_key(text: str) -> str | None:
normalized = _normalize_nepali_digits(" ".join(text.split()))
patterns = (
r"(?P<day>\d{1,2})\s+(?P<month>[^\s,]+)\s+(?P<year>\d{4})",
r"(?P<year>\d{4})\s+(?P<month>[^\s,]+)\s+(?P<day>\d{1,2})",
)
for pattern in patterns:
match = re.search(pattern, normalized)
if not match:
continue
month = NEPALI_MONTHS.get(match.group("month"))
if month is None:
continue
return f"{int(match.group('year')):04d}-{month:02d}-{int(match.group('day')):02d}"
return None
def _normalize_nepali_digits(text: str) -> str:
table = str.maketrans("०१२३४५६७८९", "0123456789")
return text.translate(table)
def _clean_date_label(text: str) -> str:
return " ".join(text.split()).strip()
def _dedupe_publication_items(items: list[NayaNepalPublicationItem]) -> list[NayaNepalPublicationItem]:
seen: set[tuple[str | None, str | None, str, str | None]] = set()
unique: list[NayaNepalPublicationItem] = []
for item in items:
key = (item.article_url, item.image_url, item.title, item.date_key)
if key in seen:
continue
seen.add(key)
unique.append(item)
return unique
def _language_page_alignment(
item: NayaNepalPublicationItem,
epaper: GorkhapatraEpaperAsset | None,
pdf_path: str | None,
pdf_sha256: str | None,
hits: list[GorkhapatraLanguagePageHit],
status: str,
warnings: list[str],
) -> GorkhapatraLanguagePageAlignment:
return GorkhapatraLanguagePageAlignment(
publication_item=item,
epaper=epaper,
pdf_path=pdf_path,
pdf_sha256=pdf_sha256,
page_hits=hits,
alignment_status=status,
claim_evidence_eligible=False,
warnings=warnings,
)
def _alignment_source_url(alignment: dict[str, Any]) -> str | None:
epaper = alignment.get("epaper")
if isinstance(epaper, dict):
return _clean_optional_str(epaper.get("direct_pdf_url")) or _clean_optional_str(epaper.get("viewer_url"))
publication_item = alignment.get("publication_item")
if isinstance(publication_item, dict):
return _clean_optional_str(publication_item.get("article_url")) or _clean_optional_str(publication_item.get("image_url"))
return None
def _gorkhapatra_language_page_audit(
*,
publication_sources: list[str],
epaper_sources: list[str],
publication_items: list[NayaNepalPublicationItem],
alignments: list[GorkhapatraLanguagePageAlignment],
warnings: list[str],
) -> GorkhapatraLanguagePageAudit:
status_counts: dict[str, int] = {}
language_counts: dict[str, int] = {}
for alignment in alignments:
status_counts[alignment.alignment_status] = status_counts.get(alignment.alignment_status, 0) + 1
language = alignment.publication_item.language
if language:
language_counts[language] = language_counts.get(language, 0) + 1
return GorkhapatraLanguagePageAudit(
publication_sources=publication_sources,
epaper_sources=epaper_sources,
publication_item_count=len(publication_items),
aligned_count=sum(1 for alignment in alignments if alignment.epaper is not None),
language_page_hit_count=sum(len(alignment.page_hits) for alignment in alignments),
status_counts=dict(sorted(status_counts.items())),
language_counts=dict(sorted(language_counts.items())),
alignments=alignments,
warnings=warnings,
)
def _gorkhapatra_language_page_pack_summary(
manifest_path: Path,
entries: list[ManifestEntry],
*,
copied_page_count: int,
warnings: list[str],
) -> GorkhapatraLanguagePagePackSummary:
language_counts: dict[str, int] = {}
reference_status_counts: dict[str, int] = {}
for entry in entries:
language = entry.metadata.get("language")
if isinstance(language, str) and language:
language_counts[language] = language_counts.get(language, 0) + 1
status = str(entry.metadata.get("reference_status") or "unknown")
reference_status_counts[status] = reference_status_counts.get(status, 0) + 1
return GorkhapatraLanguagePagePackSummary(
manifest_path=str(manifest_path),
sample_count=len(entries),
copied_page_count=copied_page_count,
language_counts=dict(sorted(language_counts.items())),
reference_status_counts=dict(sorted(reference_status_counts.items())),
warnings=warnings,
)
def _dedupe_articles(articles: list[GorkhapatraArticleAsset]) -> list[GorkhapatraArticleAsset]:
seen: set[tuple[str | None, str | None, str]] = set()
unique: list[GorkhapatraArticleAsset] = []
for article in articles:
key = (article.article_url, article.image_url, article.title)
if key in seen:
continue
seen.add(key)
unique.append(article)
return unique
def _dedupe_epapers(epapers: list[GorkhapatraEpaperAsset]) -> list[GorkhapatraEpaperAsset]:
seen: set[str] = set()
unique: list[GorkhapatraEpaperAsset] = []
for epaper in epapers:
if epaper.direct_pdf_url in seen:
continue
seen.add(epaper.direct_pdf_url)
unique.append(epaper)
return unique
def _artifact_kind(path: Path) -> str:
suffix = path.suffix.lower()
if suffix == ".pdf":
return "pdf"
if suffix in {".jpg", ".jpeg", ".png", ".webp"}:
return "image"
return "file"
def _url_suffix(url: str, *, default: str) -> str:
suffix = Path(urlparse(url).path).suffix.lower()
if suffix in {".jpg", ".jpeg", ".png", ".webp", ".pdf"}:
return suffix
return default
def _resolve_audit_artifact_path(path_value: str, audit_json_path: Path) -> Path:
path = Path(path_value)
if path.is_absolute():
return path
audit_dir = audit_json_path.parent
candidate = audit_dir / path
if candidate.exists():
return candidate
return path
def _write_pending_reference(
path: Path,
*,
sample_id: str,
source_url: str | None,
language: str | None,
source_kind: str,
) -> None:
payload = {
"text": "",
"reading_order": [],
"tables": [],
"figures": [],
"metadata": {
"sample_id": sample_id,
"source_url": source_url,
"language": language,
"source_kind": source_kind,
"reference_status": "pending_manual_label",
"claim_evidence_eligible": False,
"notes": "Real Gorkhapatra source input. Text/layout reference must be manually verified before claim use.",
},
}
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
def _render_pdf_pages(source_pdf: Path, inputs_dir: Path, *, epaper_index: int, max_pages: int) -> list[tuple[int, Path]]:
try:
import fitz # type: ignore[import-not-found]
except Exception as exc: # noqa: BLE001 - optional dependency.
raise DataValidationError(f"PDF page rendering requires PyMuPDF: {type(exc).__name__}: {exc}") from exc
rendered: list[tuple[int, Path]] = []
with fitz.open(source_pdf) as document:
page_count = min(document.page_count, max_pages)
for page_index in range(page_count):
page = document[page_index]
target = inputs_dir / f"gorkhapatra-epaper-{epaper_index:04d}-page-{page_index + 1:04d}.png"
pixmap = page.get_pixmap(matrix=fitz.Matrix(2.0, 2.0), alpha=False)
pixmap.save(target)
rendered.append((page_index, target))
return rendered
def _gorkhapatra_slices(language: str | None, source_kind: str) -> list[str]:
slices = {"gorkhapatra", "naya_nepal", "real_publication", "image"}
if source_kind == "article_image":
slices.add("article_image")
if source_kind == "epaper_page":
slices.update({"epaper", "page"})
if source_kind == "language_page_candidate":
slices.update({"epaper", "page", "language_page_candidate", "pending_reference"})
if language:
slices.add(f"language:{language}")
return sorted(slices)
def _gorkhapatra_pack_summary(
manifest_path: Path,
entries: list[ManifestEntry],
*,
article_image_count: int,
epaper_page_count: int,
warnings: list[str],
) -> GorkhapatraPackSummary:
language_counts: dict[str, int] = {}
reference_status_counts: dict[str, int] = {}
for entry in entries:
language = entry.metadata.get("language")
if isinstance(language, str) and language:
language_counts[language] = language_counts.get(language, 0) + 1
status = str(entry.metadata.get("reference_status") or "unknown")
reference_status_counts[status] = reference_status_counts.get(status, 0) + 1
return GorkhapatraPackSummary(
manifest_path=str(manifest_path),
sample_count=len(entries),
article_image_count=article_image_count,
epaper_page_count=epaper_page_count,
language_counts=dict(sorted(language_counts.items())),
reference_status_counts=dict(sorted(reference_status_counts.items())),
warnings=warnings,
)
def _write_gorkhapatra_pack_summary(summary: GorkhapatraPackSummary, output_dir: Path) -> None:
json_path = output_dir / "gorkhapatra-pack.json"
md_path = output_dir / "gorkhapatra-pack.md"
summary.summary_json_path = str(json_path)
summary.summary_md_path = str(md_path)
json_path.write_text(json.dumps(summary.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
lines = [
"# Gorkhapatra Validation Pack",
"",
f"- manifest: `{summary.manifest_path}`",
f"- samples: {summary.sample_count}",
f"- article images: {summary.article_image_count}",
f"- epaper pages: {summary.epaper_page_count}",
"",
"## Reference Status",
"",
]
lines.extend(f"- `{key}`: {value}" for key, value in summary.reference_status_counts.items()) if summary.reference_status_counts else lines.append("- none")
lines.extend(["", "## Languages", ""])
lines.extend(f"- {language}: {count}" for language, count in summary.language_counts.items()) if summary.language_counts else lines.append("- none")
lines.extend(["", "## Warnings", ""])
lines.extend(f"- {warning}" for warning in summary.warnings) if summary.warnings else lines.append("- none")
md_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _write_gorkhapatra_language_page_pack_summary(summary: GorkhapatraLanguagePagePackSummary, output_dir: Path) -> None:
json_path = output_dir / "gorkhapatra-language-page-pack.json"
md_path = output_dir / "gorkhapatra-language-page-pack.md"
summary.summary_json_path = str(json_path)
summary.summary_md_path = str(md_path)
json_path.write_text(json.dumps(summary.to_dict(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
lines = [
"# Gorkhapatra Language Page Candidate Pack",
"",
f"- manifest: `{summary.manifest_path}`",
f"- samples: {summary.sample_count}",
f"- copied page candidates: {summary.copied_page_count}",
"",
"## Reference Status",
"",
]
lines.extend(f"- `{key}`: {value}" for key, value in summary.reference_status_counts.items()) if summary.reference_status_counts else lines.append("- none")
lines.extend(["", "## Languages", ""])
lines.extend(f"- {language}: {count}" for language, count in summary.language_counts.items()) if summary.language_counts else lines.append("- none")
lines.extend(["", "## Warnings", ""])
lines.extend(f"- {warning}" for warning in summary.warnings) if summary.warnings else lines.append("- none")
lines.extend(
[
"",
"## Claim Status",
"",
"All rows are `claim_evidence_eligible=false` and `pending_manual_label` until page-language disambiguation and reference transcription are complete.",
]
)
md_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _write_gorkhapatra_language_page_review(
summary: GorkhapatraLanguagePageReviewSummary,
rows: list[dict[str, Any]],
) -> None:
json_path = Path(summary.review_json_path)
csv_path = Path(summary.review_csv_path)
md_path = Path(summary.review_md_path)
payload = {
"summary": summary.to_dict(),
"rows": rows,
"review_decision_values": [
"accept_language_page",
"reject_wrong_language",
"reject_not_language_page",
"duplicate_candidate",
"needs_additional_review",
],
}
json_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
fieldnames = [
"sample_id",
"candidate_language",
"date_label",
"date_key",
"pdf_page_index",
"pdf_page_number",
"image_path",
"reference_path",
"article_url",
"source_image_url",
"source_url",
"pdf_path",
"pdf_sha256",
"language_page_marker",
"alignment_status",
"page_disambiguation_status",
"reference_status",
"claim_evidence_eligible",
"review_decision",
"reviewer",
"reviewed_at",
"notes",
]
with csv_path.open("w", encoding="utf-8", newline="") as handle:
writer = csv.DictWriter(handle, fieldnames=fieldnames)
writer.writeheader()
for row in rows:
writer.writerow({field: row.get(field) for field in fieldnames})
lines = [
"# Gorkhapatra Language Page Review",
"",
f"- manifest: `{summary.manifest_path}`",
f"- samples: {summary.sample_count}",
f"- pending review: {summary.pending_review_count}",
f"- claim eligible: {summary.claim_evidence_eligible_count}",
f"- CSV: `{summary.review_csv_path}`",
"",
"## Review Rule",
"",
"Rows remain non-claim-ready until a reviewer confirms the language-page match and creates a verified reference transcript/layout.",
"",
"## Languages",
"",
]
lines.extend(f"- {language}: {count}" for language, count in summary.language_counts.items()) if summary.language_counts else lines.append("- none")
lines.extend(["", "## Same Page Candidate Counts", ""])
lines.extend(f"- `{key}`: {count}" for key, count in summary.page_candidate_counts.items()) if summary.page_candidate_counts else lines.append("- none")
if summary.warnings:
lines.extend(["", "## Warnings", ""])
lines.extend(f"- {warning}" for warning in summary.warnings)
lines.extend(["", "## Rows", ""])
for row in rows[:100]:
lines.append(
f"- `{row['sample_id']}` {row.get('candidate_language') or 'unknown'} "
f"date={row.get('date_key') or 'unknown'} page={row.get('pdf_page_number') or 'unknown'} "
f"status={row.get('page_disambiguation_status') or 'unknown'}"
)
md_path.write_text("\n".join(lines) + "\n", encoding="utf-8")
def _clean_optional_str(value: Any) -> str | None:
if not isinstance(value, str):
return None
cleaned = value.strip()
return cleaned or None
"""Source audits for real-world validation data."""
from __future__ import annotations
import csv
import json
import re
import shutil
from dataclasses import dataclass, field
from html.parser import HTMLParser
from pathlib import Path
from typing import Any
from urllib.parse import parse_qs, unquote, urljoin, urlparse
from urllib.request import Request, urlopen
from .errors import DataValidationError
from .manifest import ManifestEntry, load_manifest, sha256_file, sha256_text, write_manifest
DEFAULT_GORKHAPATRA_CATEGORY_URLS = (
"https://gorkhapatraonline.com/categories/new-nepal",
"https://gorkhapatraonline.com/categories/new-nepal?page=2",
)
DEFAULT_GORKHAPATRA_EPAPER_URL = (
"https://epaper.gorkhapatraonline.com/epaper-list?"
"pdfdaterange1=2026-06-09&pdfdaterange2=2026-06-16&slug=gorkhapatra"
)
GORKHAPATRA_ONLINE_BASE = "https://gorkhapatraonline.com"
GORKHAPATRA_EPAPER_BASE = "https://epaper.gorkhapatraonline.com"
DEFAULT_NAYA_NEPAL_PUBLICATION_URL = "https://gorkhapatraonline.com/publications/naya-nepal"
LANGUAGE_PAGE_MARKERS = ("भाषा पृष्ठ", "efiff k[i7", "bhasa pristha", "bhasa prishtha")
NEPALI_MONTHS = {
"बैशाख": 1,
"वैशाख": 1,
"जेठ": 2,
"असार": 3,
"साउन": 4,
"भदौ": 5,
"भाद्र": 5,
"असोज": 6,
"कार्तिक": 7,
"मंसिर": 8,
"मङ्सिर": 8,
"पुस": 9,
"पुष": 9,
"माघ": 10,
"फागुन": 11,
"चैत": 12,
}
class DownloadedArtifact:
url: str
path: str
sha256: str
size_bytes: int
kind: str
warnings: list[str] = field(default_factory=list)
pdf_audit: dict[str, Any] | None = None
def to_dict(self) -> dict[str, Any]:
payload = {
"url": self.url,
"path": self.path,
"sha256": self.sha256,
"size_bytes": self.size_bytes,
"kind": self.kind,
"warnings": self.warnings,
}
if self.pdf_audit is not None:
payload["pdf_audit"] = self.pdf_audit
return payload
class GorkhapatraArticleAsset:
article_url: str | None
image_url: str | None
title: str
language: str | None
source_page: str
downloaded_image: DownloadedArtifact | None = None
def to_dict(self) -> dict[str, Any]:
return {
"article_url": self.article_url,
"image_url": self.image_url,
"title": self.title,
"language": self.language,
"source_page": self.source_page,
"downloaded_image": self.downloaded_image.to_dict() if self.downloaded_image else None,
}
class GorkhapatraEpaperAsset:
viewer_url: str