ad caching

app.py

@@ -42,7 +42,7 @@ class ValidationAnnotator:
     """
     
     def __init__(self, input_file: str, hf_dataset_repo: Optional[str] = None, hf_token: Optional[str] = None,
-                 pdf_dir: Optional[str] = None, pdf_url_base: Optional[str] = None):
+                 pdf_dir: Optional[str] = None, pdf_url_base: Optional[str] = None, pdf_repo_id: Optional[str] = None):
         self.input_file = Path(input_file)
         self.output_file = self.input_file.parent / f"{self.input_file.stem}_human_validated.jsonl"
         
@@ -53,6 +53,8 @@ class ValidationAnnotator:
         # PDF configuration
         self.pdf_dir = Path(pdf_dir) if pdf_dir else None
         self.pdf_url_base = pdf_url_base
+        self.pdf_repo_id = pdf_repo_id
+        
         if self.pdf_dir and not self.pdf_dir.exists():
             print(f"⚠️ PDF directory not found: {self.pdf_dir}")
         self.hf_enabled = False
@@ -471,6 +473,28 @@ class ValidationAnnotator:
                 print(f"📄 Found PDF for sample {self.current_idx}: {pdf_value} (Page {page_num})", flush=True)
             else:
                 print(f"⚠️ PDF file not found: {pdf_path}", flush=True)
+        elif source_doc and self.pdf_repo_id:
+            # Server-side caching via HF Hub (avoids CORS/frontend download issues)
+            # Remove leading slash if present
+            source_doc_clean = source_doc.lstrip('/')
+            try:
+                from huggingface_hub import hf_hub_download
+                print(f"📥 Downloading/Caching PDF from {self.pdf_repo_id}: {source_doc_clean}", flush=True)
+                pdf_path_cached = hf_hub_download(
+                    repo_id=self.pdf_repo_id,
+                    filename=source_doc_clean,
+                    repo_type="dataset",
+                    token=self.hf_token
+                )
+                pdf_value = str(pdf_path_cached)
+                print(f"📦 Cached local path: {pdf_value}", flush=True)
+            except Exception as e:
+                print(f"❌ Failed to download PDF: {e}", flush=True)
+                # Fallback to URL if download fails and url base is available
+                if self.pdf_url_base:
+                    pdf_value = f"{self.pdf_url_base.rstrip('/')}/{source_doc_clean}"
+                    print(f"⚠️ Falling back to remote URL: {pdf_value}", flush=True)
+        
         elif source_doc and self.pdf_url_base:
             # Remote PDF via URL (e.g., HF Datasets)
             # Remove any leading slashes from source_doc
@@ -549,9 +573,9 @@ class ValidationAnnotator:
 
 
 def create_app(input_file: str, hf_dataset_repo: Optional[str] = None, hf_token: Optional[str] = None,
-               pdf_dir: Optional[str] = None, pdf_url_base: Optional[str] = None):
+               pdf_dir: Optional[str] = None, pdf_url_base: Optional[str] = None, pdf_repo_id: Optional[str] = None):
     """Create and configure Gradio app."""
-    annotator = ValidationAnnotator(input_file, hf_dataset_repo, hf_token, pdf_dir, pdf_url_base)
+    annotator = ValidationAnnotator(input_file, hf_dataset_repo, hf_token, pdf_dir, pdf_url_base, pdf_repo_id)
     
     # Custom CSS for the green button and dark mode toggle
     css = """
@@ -943,6 +967,7 @@ if __name__ == "__main__":
     pdf_url_base = args.pdf_url_base
     
     # If no explicit PDF source, check for HF PDF repo environment variable
+    pdf_repo_id = None
     if not pdf_dir and not pdf_url_base:
         hf_pdf_repo = os.getenv("HF_RELIEFWEB_PDFS_REPO")  # e.g., "ai4data/reliefweb-pdfs"
         if hf_pdf_repo:
@@ -951,15 +976,20 @@ if __name__ == "__main__":
                 # Already a full URL, use it directly (ensure it ends with /)
                 pdf_url_base = hf_pdf_repo.rstrip('/') + '/'
             else:
-                # Repo ID format, construct the URL
+                # Repo ID format - enabling server-side caching!
+                pdf_repo_id = hf_pdf_repo
+                # Also set url base as fallback
                 pdf_url_base = f"https://huggingface.co/datasets/{hf_pdf_repo}/resolve/main/"
+            
             print(f"🌐 Using HF PDF repository: {hf_pdf_repo}", flush=True)
-            print(f"   PDF URL base: {pdf_url_base}", flush=True)
+            if pdf_repo_id:
+                print(f"   🚀 Server-side caching ENABLED for repo: {pdf_repo_id}", flush=True)
+            print(f"   PDF URL base (fallback): {pdf_url_base}", flush=True)
         else:
             print("⚠️ No PDF source configured. Set --pdf-dir, --pdf-url-base, or HF_RELIEFWEB_PDFS_REPO.", flush=True)
 
     # Create and launch the app
-    app = create_app(input_file, hf_dataset_repo, hf_token, pdf_dir, pdf_url_base)
+    app = create_app(input_file, hf_dataset_repo, hf_token, pdf_dir, pdf_url_base, pdf_repo_id)
     
     # Ensure allowed paths are absolute for Gradio (only needed for local files)
     allowed = []
@@ -968,6 +998,15 @@ if __name__ == "__main__":
         allowed = [pdf_dir_parent]
         print(f"🚀 Launching with allowed_paths: {allowed}", flush=True)
         print(f"📂 PDF Directory Check: {Path(pdf_dir).exists()}", flush=True)
+    elif pdf_repo_id:
+        # If caching from HF, we need to allow access to the HF cache directory
+        # Typical path: ~/.cache/huggingface/hub
+        # We'll allow the user's home directory to be safe/simple for now, 
+        # or we could try to resolve the specific cache path.
+        # Allowing hierarchy up to home is usually robust for local caches.
+        home_dir = str(Path.home().resolve())
+        allowed = [home_dir]
+        print(f"🚀 Launching with cached HF PDFs - Allowing access to: {allowed}", flush=True)
     else:
         print("🚀 Launching with remote PDF URLs (no local allowed_paths needed)", flush=True)