feat: re-run Helping Hands with post-processing and scale sample PDFs to 12 documents

public/sample_pdfs/2025 03 SEIS ukraine_socio-economic_researchpaper.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:329a144b0e723f56e7a92cd2d8dfd9ea079f4b72e53ce438f60a1dad548e2fca
+size 1645202

public/sample_pdfs/Analysis of the impact of refugees from Ukraine on the economy of Poland.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b28a968cc33c174aa7a3d0a1d9cd5085a0abbc903c22b291343ae672028d6fd
+size 4130712

public/sample_pdfs/LCRP_2021.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b9cf25fb6c4de71347f8e4ca4ca8f1a3c38a117d3579e47b9d0f765f017a858
+size 6052678

public/sample_pdfs/LCRP_2025.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a43d3516a7167905824d3af1ca73e2386bc5be4dd95c3c3eabfa0cf14ed02a92
+size 8826747

public/sample_pdfs/LebanonResp2025.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a43d3516a7167905824d3af1ca73e2386bc5be4dd95c3c3eabfa0cf14ed02a92
+size 8826747

public/sample_pdfs/NAVIGATING HEALTH AND WELL-BEING CHALLENGES FOR REFUGEES FROM UKRAINE - 2nd Edition.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6ab3d2ffa6da1f73a0f86b6ef8c5d82764852dc9f52521bb623233d10dcea4e3
+size 2795909

public/sample_pdfs/Poland Analysis of the impact of refugees from Ukraine on the economy of Poland – March 2024.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:886eaed5ee0626e201b9758b1e20107aaf73b3e75fe498fbbc693b151230cdf1
+size 4932124

public/sample_pdfs/Revised Ukraine Situation 2025 2026 RRP FINAL (5).pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bf0fca50ffdef6653f70151422acc0a8252dcf69bf91eb13e765db1dee413684
+size 6135356

public/sample_pdfs/SESRE_WBG.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3b5a3405d248c5ae0b6f486bf7132edefc3615d2b2f2f7219ddea5dfe4fa8ff6
+size 9205502

public/sample_pdfs/Somalia_Baseline.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cdae67484a86a004e61d50b866445f57473822c9749dfcca21562d3a6a86067a
+size 1370622

public/sample_pdfs/Somalia_NSP.pdf

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9b1e81e04cfdea69865a61c2d59291ee91c596c597d5278612a8e697ee571a22
+size 92582

scripts/extract_sample_pdfs.py

@@ -17,6 +17,77 @@ EXTRACTIONS_DIR = APP_ROOT / "annotation_data" / "sample_extractions"
 LINKS_DIR = APP_ROOT / "annotation_data" / "sample_data"
 LINKS_FILE = LINKS_DIR / "sample_pdf_links.json"
 
+def merge_existing_annotations(dest_file, new_pages_data):
+    """
+    Reads existing pages data from dest_file, extracts human annotations
+    and validations, and merges them into new_pages_data.
+    """
+    if not dest_file.exists():
+        return new_pages_data
+
+    try:
+        with open(dest_file, "r", encoding="utf-8") as f:
+            old_pages_data = json.load(f)
+    except Exception as e:
+        print(f"  Warning: Could not read existing file {dest_file} for merging: {e}")
+        return new_pages_data
+
+    # Map page number to its human annotations and validations
+    page_merges = {}
+    for old_page in old_pages_data:
+        page_num = old_page.get("document", {}).get("pages", [0])[0]
+        human_added = []
+        val_map = {} # dataset_name (lower) -> validations list
+        
+        for ds in old_page.get("datasets", []):
+            ds_name = ds.get("dataset_name", {})
+            name_text = ds_name.get("text", "") if isinstance(ds_name, dict) else str(ds_name)
+            if not name_text:
+                continue
+            
+            # Check if it is human added
+            is_human = (ds.get("source") == "human" or ds.get("annotator") is not None)
+            if is_human:
+                human_added.append(ds)
+            
+            # Check if it has human validations
+            vals = ds.get("validations", [])
+            if vals:
+                val_map[name_text.lower()] = vals
+                
+        page_merges[page_num] = {
+            "human_added": human_added,
+            "val_map": val_map
+        }
+
+    # Merge into new_pages_data
+    for new_page in new_pages_data:
+        page_num = new_page.get("document", {}).get("pages", [0])[0]
+        merge_info = page_merges.get(page_num)
+        if not merge_info:
+            continue
+            
+        # 1. Restore validations for model-extracted datasets
+        for ds in new_page.get("datasets", []):
+            ds_name = ds.get("dataset_name", {})
+            name_text = ds_name.get("text", "") if isinstance(ds_name, dict) else str(ds_name)
+            if name_text.lower() in merge_info["val_map"]:
+                ds["validations"] = merge_info["val_map"][name_text.lower()]
+
+        # 2. Append human-added datasets
+        existing_names = {
+            (ds.get("dataset_name", {}).get("text", "") if isinstance(ds.get("dataset_name"), dict) else str(ds.get("dataset_name"))).lower()
+            for ds in new_page.get("datasets", [])
+        }
+        for ds in merge_info["human_added"]:
+            ds_name = ds.get("dataset_name", {})
+            name_text = ds_name.get("text", "") if isinstance(ds_name, dict) else str(ds_name)
+            if name_text.lower() not in existing_names:
+                new_page["datasets"].append(ds)
+                existing_names.add(name_text.lower())
+
+    return new_pages_data
+
 def main():
     if not PDFS_DIR.exists():
         print(f"Error: {PDFS_DIR} does not exist.")
@@ -42,6 +113,33 @@ def main():
         doc_name = f"doc_{i}"
         print(f"\n[{i}/{len(pdf_files)}] Processing: {pdf_path.name} -> {doc_name}")
 
+        dest_dir = EXTRACTIONS_DIR / doc_name / "raw"
+        dest_file = dest_dir / f"{doc_name}_direct_judged.jsonl"
+
+        if dest_file.exists() and doc_name != "doc_1":
+            print(f"  Existing extraction file found at: {dest_file}")
+            print("  Skipping model extraction and reading existing file...")
+            try:
+                with open(dest_file, "r", encoding="utf-8") as f:
+                    pages_data = json.load(f)
+                num_pages = len(pages_data)
+                has_ds = any(len(p.get("datasets", [])) > 0 for p in pages_data)
+                link_entry = {
+                    "index": i,
+                    "src_docname": doc_name,
+                    "original_name": pdf_path.stem,
+                    "landing_page_url": f"/sample_pdfs/{pdf_path.name}",
+                    "direct_pdf_url": f"/sample_pdfs/{pdf_path.name}",
+                    "status": "success",
+                    "has_revalidation": True,
+                    "has_datasets": has_ds,
+                    "num_pages": num_pages
+                }
+                links.append(link_entry)
+                continue
+            except Exception as e:
+                print(f"  Error reading existing file: {e}. Re-running extraction...")
+
         try:
             doc = fitz.open(str(pdf_path))
             num_pages = len(doc)
@@ -58,7 +156,7 @@ def main():
                 result = extractor.extract_from_text(
                     text,
                     include_confidence=True,
-                    use_classifier=False,
+                    use_classifier=True,
                     extract_provenance=True,
                     dataset_threshold=0.3,
                     apply_heuristics=True,
@@ -110,6 +208,11 @@ def main():
             dest_dir.mkdir(parents=True, exist_ok=True)
             dest_file = dest_dir / f"{doc_name}_direct_judged.jsonl"
             
+            if doc_name == "doc_1" and dest_file.exists():
+                print(f"  Merging existing annotations/validations for {doc_name}...")
+                pages_data = merge_existing_annotations(dest_file, pages_data)
+                has_ds = any(len(p.get("datasets", [])) > 0 for p in pages_data)
+            
             with open(dest_file, "w", encoding="utf-8") as f:
                 json.dump(pages_data, f, indent=2, ensure_ascii=False)
 

upload_to_hf.py

@@ -3,6 +3,15 @@ from pathlib import Path
 from huggingface_hub import HfApi, create_repo
 
 token = os.environ.get("HF_TOKEN")
+if not token:
+    # Try reading from .env file
+    env_path = Path(__file__).resolve().parent / ".env"
+    if env_path.exists():
+        for line in env_path.read_text().splitlines():
+            if line.startswith("HF_TOKEN="):
+                token = line.split("=", 1)[1].strip()
+                break
+
 if not token:
     raise ValueError("HF_TOKEN not found in environment.")