Upload 8 files

.env.template

@@ -0,0 +1,7 @@
+# Fill for local development only. On HF Spaces, use Secrets instead.
+OPENAI_API_KEY=
+DEEPSEEK_API_KEY=
+GOOGLE_API_KEY=
+GOOGLE_CSE_ID=
+OPENAI_MODEL=gpt-4o-mini
+DEEPSEEK_MODEL=deepseek-chat

README.md

@@ -0,0 +1,85 @@
+# 🔎 HF Space — Automated Requirement Citation Checker
+
+This Space ingests a CSV with columns: **Prompt, Rubric, ID, Requirements, Important Evaluation Rules** and
+verifies each atomic claim in the **Requirements** column against the public web.
+
+It searches via **Google Custom Search (CSE)**, fetches top pages, and asks **both OpenAI** and **DeepSeek**
+LLMs to decide if each claim is **SUPPORTED**, **CONTRADICTED**, or **UNVERIFIABLE**, returning citations and a concise reason.
+A per-row conclusion is computed from all claims.
+
+## Quick Start (Hugging Face Spaces)
+
+1. Create a new Space → SDK: **Gradio** → Visibility: your choice.
+2. Upload these files to the Space root:
+   - `app.py`
+   - `verifier.py`
+   - `search.py`
+   - `models.py`
+   - `utils.py`
+   - `requirements.txt`
+   - `.env.template` (optional for local dev)
+3. In the Space page → **Settings** → **Secrets**, add:
+   - `OPENAI_API_KEY` — your OpenAI key (you mentioned `OAPI1`).
+   - `DEEPSEEK_API_KEY` — your DeepSeek key (you mentioned `DAPI`).
+   - `GOOGLE_API_KEY` — Google API key (you mentioned `API1`).
+   - `GOOGLE_CSE_ID` — your CSE ID (you mentioned `CX1`).
+   - (Optional) `OPENAI_MODEL` (default `gpt-4o-mini`)
+   - (Optional) `DEEPSEEK_MODEL` (default `deepseek-chat`)
+4. Deploy. Open the Space, upload your CSV, and click **Run Verification**.
+
+## Output
+
+- **Table view** summarizing Supported/Contradicted/Unverifiable counts and overall conclusion per row.
+- **JSON** with full detail (per-claim statuses, reasons, citations, and both models' raw decisions).
+- **Markdown report** with human-readable results and links.
+
+## CSV Format
+
+Columns required (case-sensitive):
+- `Prompt`
+- `Rubric`
+- `ID`
+- `Requirements`
+- `Important Evaluation Rules`
+
+Only **Requirements** is fact-checked; other columns are passed through for context if you want to extend prompts later.
+
+## How it works
+
+- `utils.split_atomic_claims` heuristically splits Requirements text into atomic claims (lists + sentences).
+- `search.google_cse` queries Google CSE (multiple queries per claim), collects unique result links.
+- `search.fetch_page` downloads pages and extracts plain text + titles for LLM review.
+- `verifier.verify_claim` builds a single packed prompt with all sources and asks **both** OpenAI & DeepSeek to return structured JSON:
+  ```json
+  {
+    "status": "SUPPORTED|CONTRADICTED|UNVERIFIABLE",
+    "reason": "brief rationale",
+    "citations": [{"url": "...", "title": "...", "quote": "short"}]
+  }
+  ```
+- An ensemble chooses the final status; the other model’s output is preserved for transparency.
+- `verifier.verify_requirement` aggregates claim-level results and computes a row-level conclusion:
+  - Any CONTRADICTED → **CONTRADICTED**
+  - Else if all supported → **SUPPORTED**
+  - Else if mixed → **PARTIALLY SUPPORTED**
+  - Else → **UNVERIFIABLE**
+
+## Local Development
+
+```bash
+python -m venv .venv && source .venv/bin/activate
+pip install -r requirements.txt
+cp .env.template .env  # fill keys
+python app.py
+```
+
+## Notes & Tips
+
+- Be conservative: the models are instructed to mark **UNVERIFIABLE** when evidence is insufficient.
+- You can tune the number of search results and page length limits in `search.py`.
+- For highly technical domains, consider whitelisting official domains in your CSE for higher precision.
+- If you hit rate limits, Tenacity will retry with exponential backoff.
+
+## License
+
+MIT

app.py

@@ -0,0 +1,96 @@
+import os, io, json, asyncio, pandas as pd, gradio as gr
+from verifier import verify_requirement
+from dotenv import load_dotenv
+
+load_dotenv()
+
+INTRO_MD = """
+# 🔎 Automatic Citation & Requirement Checker
+
+Upload a CSV with columns: **Prompt, Rubric, ID, Requirements, Important Evaluation Rules**.  
+This Space will extract atomic claims from each **Requirements** cell, search the web (Google CSE), fetch pages, and ask **both OpenAI and DeepSeek** models to verify each claim with citations.  
+At the end of each row, you'll get a summary and conclusion.
+
+**Secrets**: Configure in the Space settings (Repository → Settings → Secrets):  
+- `OPENAI_API_KEY` (your OpenAI key)  
+- `DEEPSEEK_API_KEY` (your DeepSeek key)  
+- `GOOGLE_API_KEY` (your Google API key)  
+- `GOOGLE_CSE_ID` (your Custom Search Engine ID)  
+(Optional) `OPENAI_MODEL`, `DEEPSEEK_MODEL`
+"""
+
+async def process_csv(file_obj) -> tuple:
+    df = pd.read_csv(file_obj)
+    required_cols = ["Prompt","Rubric","ID","Requirements","Important Evaluation Rules"]
+    missing = [c for c in required_cols if c not in df.columns]
+    if missing:
+        raise gr.Error(f"Missing required columns: {missing}")
+    rows = []
+    for idx, row in df.iterrows():
+        req_text = str(row["Requirements"] or "").strip()
+        rid = row.get("ID", idx)
+        if not req_text:
+            rows.append({"ID": rid, "status":"SKIPPED (empty Requirements)", "result": {}})
+            continue
+        result = await verify_requirement(req_text)
+        rows.append({"ID": rid, "status": result["conclusion"], "result": result})
+
+    # Build outputs
+    # 1) table view
+    out_rows = []
+    for r in rows:
+        rid = r["ID"]
+        status = r["status"]
+        if not r["result"]:
+            out_rows.append({"ID": rid, "Conclusion": status, "Supported":0, "Contradicted":0, "Unverifiable":0, "Claims":0})
+            continue
+        counts = r["result"]["summary"]
+        out_rows.append({
+            "ID": rid,
+            "Conclusion": status,
+            "Supported": counts.get("SUPPORTED",0),
+            "Contradicted": counts.get("CONTRADICTED",0),
+            "Unverifiable": counts.get("UNVERIFIABLE",0),
+            "Claims": sum(counts.values())
+        })
+    table = pd.DataFrame(out_rows)
+
+    # 2) detailed JSON
+    detailed = {"rows": rows}
+    json_bytes = io.BytesIO(json.dumps(detailed, indent=2, ensure_ascii=False).encode("utf-8"))
+    json_bytes.name = "verification_results.json"
+
+    # 3) pretty markdown report
+    md_lines = ["# Results\n"]
+    for r in rows:
+        rid = r["ID"]
+        md_lines.append(f"## ID: {rid}\n**Conclusion**: {r['status']}\n")
+        if not r["result"]:
+            continue
+        for claimres in r["result"]["claims"]:
+            md_lines.append(f"- **Claim**: {claimres['claim']}\n  - Status: {claimres['status']}\n  - Reason: {claimres.get('reason','')}\n")
+            cites = claimres.get("citations", [])[:4]
+            if cites:
+                for c in cites:
+                    md_lines.append(f"    - [{c.get('title','source')}]({c.get('url','')}) — “{c.get('quote','')}”")
+        md_lines.append("")
+    report_md = "\n".join(md_lines)
+    report_bytes = io.BytesIO(report_md.encode("utf-8"))
+    report_bytes.name = "report.md"
+    return table, json_bytes, report_bytes
+
+with gr.Blocks(fill_height=True) as demo:
+    gr.Markdown(INTRO_MD)
+    with gr.Row():
+        file = gr.File(label="Upload CSV", file_types=[".csv"], type="binary")
+    run = gr.Button("Run Verification", variant="primary")
+    with gr.Row():
+        table = gr.Dataframe(headers=["ID","Conclusion","Supported","Contradicted","Unverifiable","Claims"], interactive=False)
+    with gr.Row():
+        json_out = gr.File(label="Download JSON")
+        md_out = gr.File(label="Download Markdown Report")
+
+    run.click(fn=process_csv, inputs=[file], outputs=[table, json_out, md_out])
+
+if __name__ == "__main__":
+    demo.launch()

models.py

@@ -0,0 +1,47 @@
+import os
+import json
+import asyncio
+from typing import Dict, Any, Optional, List
+import httpx
+from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
+
+OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
+DEEPSEEK_API_KEY = os.getenv("DEEPSEEK_API_KEY")
+DEEPSEEK_BASE_URL = os.getenv("DEEPSEEK_BASE_URL", "https://api.deepseek.com/v1")
+OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL", "https://api.openai.com/v1")
+OPENAI_MODEL = os.getenv("OPENAI_MODEL", "gpt-4o-mini")
+DEEPSEEK_MODEL = os.getenv("DEEPSEEK_MODEL", "deepseek-chat")
+
+HEADERS_OAI = {"Authorization": f"Bearer {OPENAI_API_KEY}", "Content-Type": "application/json"}
+HEADERS_DSK = {"Authorization": f"Bearer {DEEPSEEK_API_KEY}", "Content-Type": "application/json"}
+
+class TransientError(Exception):
+    pass
+
+@retry(stop=stop_after_attempt(4), wait=wait_exponential(multiplier=1, min=1, max=20),
+       retry=retry_if_exception_type(TransientError))
+async def call_openai(messages: List[Dict[str, str]], temperature: float = 0.0, max_tokens: int = 1200) -> str:
+    if not OPENAI_API_KEY:
+        raise RuntimeError("OPENAI_API_KEY not set")
+    payload = {"model": OPENAI_MODEL, "messages": messages, "temperature": temperature, "max_tokens": max_tokens}
+    async with httpx.AsyncClient(timeout=60.0) as client:
+        r = await client.post(f"{OPENAI_BASE_URL}/chat/completions", headers=HEADERS_OAI, json=payload)
+        if r.status_code >= 500:
+            raise TransientError(f"OpenAI transient {r.status_code}")
+        r.raise_for_status()
+        data = r.json()
+        return data["choices"][0]["message"]["content"]
+
+@retry(stop=stop_after_attempt(4), wait=wait_exponential(multiplier=1, min=1, max=20),
+       retry=retry_if_exception_type(TransientError))
+async def call_deepseek(messages: List[Dict[str, str]], temperature: float = 0.0, max_tokens: int = 1200) -> str:
+    if not DEEPSEEK_API_KEY:
+        raise RuntimeError("DEEPSEEK_API_KEY not set")
+    payload = {"model": DEEPSEEK_MODEL, "messages": messages, "temperature": temperature, "max_tokens": max_tokens}
+    async with httpx.AsyncClient(timeout=60.0) as client:
+        r = await client.post(f"{DEEPSEEK_BASE_URL}/chat/completions", headers=HEADERS_DSK, json=payload)
+        if r.status_code >= 500:
+            raise TransientError(f"DeepSeek transient {r.status_code}")
+        r.raise_for_status()
+        data = r.json()
+        return data["choices"][0]["message"]["content"]

requirements.txt

@@ -0,0 +1,7 @@
+gradio>=4.44.0
+pandas>=2.2.0
+python-dotenv>=1.0.1
+httpx>=0.27.0
+tenacity>=8.3.0
+beautifulsoup4>=4.12.3
+html5lib>=1.1

search.py

@@ -0,0 +1,60 @@
+import os, re, asyncio, html
+from typing import List, Dict, Any, Tuple
+import httpx
+from bs4 import BeautifulSoup
+from tenacity import retry, stop_after_attempt, wait_exponential, retry_if_exception_type
+
+GOOGLE_CSE_ID = os.getenv("GOOGLE_CSE_ID")
+GOOGLE_API_KEY = os.getenv("GOOGLE_API_KEY")
+
+UA = "Mozilla/5.0 (X11; Linux x86_64) CitationChecker/1.0 (+https://hf.space)"
+HEADERS = {"User-Agent": UA, "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"}
+
+class TransientError(Exception):
+    pass
+
+def clean_text(t: str) -> str:
+    t = re.sub(r"\s+", " ", t or "").strip()
+    return t
+
+@retry(stop=stop_after_attempt(4), wait=wait_exponential(min=1, max=20), retry=retry_if_exception_type(TransientError))
+async def google_cse(query: str, num: int = 6) -> List[Dict[str, Any]]:
+    if not GOOGLE_CSE_ID or not GOOGLE_API_KEY:
+        raise RuntimeError("GOOGLE_CSE_ID/GOOGLE_API_KEY not set")
+    url = "https://www.googleapis.com/customsearch/v1"
+    params = {"q": query, "cx": GOOGLE_CSE_ID, "key": GOOGLE_API_KEY, "num": min(num, 10)}
+    async with httpx.AsyncClient(timeout=60.0, headers=HEADERS) as client:
+        r = await client.get(url, params=params)
+        if r.status_code >= 500:
+            raise TransientError(f"Google CSE transient {r.status_code}")
+        r.raise_for_status()
+        data = r.json()
+        items = data.get("items", [])
+        results = []
+        for it in items:
+            results.append({
+                "title": it.get("title"),
+                "link": it.get("link"),
+                "snippet": it.get("snippet"),
+                "displayLink": it.get("displayLink"),
+            })
+        return results
+
+@retry(stop=stop_after_attempt(4), wait=wait_exponential(min=1, max=20), retry=retry_if_exception_type(TransientError))
+async def fetch_page(url: str) -> Dict[str, str]:
+    async with httpx.AsyncClient(timeout=60.0, headers=HEADERS) as client:
+        r = await client.get(url, follow_redirects=True)
+        if r.status_code >= 500:
+            raise TransientError(f"Fetch transient {r.status_code}")
+        r.raise_for_status()
+        html_text = r.text
+    soup = BeautifulSoup(html_text, "html.parser")
+    title = soup.title.get_text(strip=True) if soup.title else ""
+    # Extract text content (limit size to avoid token bloat)
+    for s in soup(["script", "style", "noscript"]):
+        s.extract()
+    text = soup.get_text(separator=" ")
+    text = re.sub(r"\s+", " ", text).strip()
+    if len(text) > 25000:
+        text = text[:25000]
+    return {"url": url, "title": title, "text": text}

utils.py

@@ -0,0 +1,35 @@
+import re, json
+from typing import List, Dict
+
+def split_atomic_claims(requirement_text: str) -> List[str]:
+    """
+    Heuristics to split requirement text into atomic factual claims.
+    Falls back to sentence splitting and numbered/bulleted lists.
+    """
+    text = requirement_text.strip()
+    # Split on list numbers/bullets
+    parts = re.split(r"(?:^\s*[-*]\s+|^\s*\d+\.\s+|\n\s*[-*]\s+|\n\s*\d+\.\s+)", text, flags=re.M)
+    # Fallback to sentences if needed
+    claims = []
+    for p in parts:
+        p = p.strip(" •;-–—")
+        if not p:
+            continue
+        # Split by sentence enders if the chunk is long
+        if len(p) > 220:
+            subs = re.split(r"(?<=[.!?])\s+", p)
+            for s in subs:
+                s = s.strip()
+                if 3 < len(s) < 240:
+                    claims.append(s)
+        else:
+            claims.append(p)
+    # De-duplicate
+    seen = set()
+    uniq = []
+    for c in claims:
+        c2 = re.sub(r"\s+", " ", c).strip().rstrip(".")
+        if c2 and c2.lower() not in seen:
+            seen.add(c2.lower())
+            uniq.append(c2)
+    return uniq[:20]  # cap to keep work bounded

verifier.py

@@ -0,0 +1,99 @@
+import asyncio, json, re
+from typing import List, Dict, Any, Tuple
+from search import google_cse, fetch_page, clean_text
+from models import call_openai, call_deepseek
+from utils import split_atomic_claims
+
+VERIFY_SYSTEM = """You are a meticulous fact-checker. Given a CLAIM and a list of web SOURCES (title, url, excerpt),
+decide if the claim is SUPPORTED, CONTRADICTED, or UNVERIFIABLE. Cite the strongest 1–4 sources that support or contradict.
+If sources conflict, prefer primary/official sources and more recent data. Return strict JSON with keys:
+{{
+  "status": "SUPPORTED|CONTRADICTED|UNVERIFIABLE",
+  "reason": "brief rationale",
+  "citations": [{{"url": "...", "title": "...", "quote": "verbatim short quote"}}]
+}}
+Keep quotes under 280 characters. Be conservative: if uncertain, UNVERIFIABLE.
+"""
+
+def build_messages(claim: str, sources: List[Dict[str, str]]) -> List[Dict[str, str]]:
+    packed = []
+    for s in sources[:8]:
+        excerpt = s.get("text", "")[:1400]
+        packed.append(f"- {s.get('title','')} | {s.get('url','')}\n{excerpt}")
+    user = f"CLAIM: {claim}\n\nSOURCES:\n" + "\n\n".join(packed)
+    return [
+        {"role": "system", "content": VERIFY_SYSTEM},
+        {"role": "user", "content": user}
+    ]
+
+async def verify_claim(claim: str) -> Dict[str, Any]:
+    # 1) craft queries
+    queries = [claim, f"verify: {claim}", f"site:.gov OR site:.edu {claim[:80]}"]
+    # 2) search and fetch
+    search_tasks = [google_cse(q, num=6) for q in queries]
+    search_groups = await asyncio.gather(*search_tasks, return_exceptions=True)
+    candidates = []
+    seen = set()
+    for group in search_groups:
+        if isinstance(group, Exception): 
+            continue
+        for item in group:
+            u = item["link"]
+            if u not in seen and not any(b in u for b in ["webcache.googleusercontent.com", "translate.google"]):
+                seen.add(u)
+                candidates.append({"title": item.get("title",""), "url": u, "snippet": item.get("snippet","")})
+    fetch_tasks = [fetch_page(c["url"]) for c in candidates[:10]]
+    pages = await asyncio.gather(*fetch_tasks, return_exceptions=True)
+    sources = []
+    for i, p in enumerate(pages):
+        if isinstance(p, Exception): 
+            continue
+        # attach snippet for flavor
+        p["snippet"] = candidates[i].get("snippet","")
+        sources.append(p)
+
+    # 3) LLM verification with both vendors in parallel
+    messages = build_messages(claim, sources)
+    oai_task = call_openai(messages)
+    dsk_task = call_deepseek(messages)
+    oai_resp, dsk_resp = await asyncio.gather(oai_task, dsk_task)
+
+    def parse_json(s: str) -> Dict[str, Any]:
+        try:
+            # try to find the first JSON block
+            m = re.search(r"\{.*\}", s, re.S)
+            if m:
+                return json.loads(m.group(0))
+        except Exception:
+            pass
+        return {"status":"UNVERIFIABLE","reason":"Could not parse model output","citations":[]}
+
+    o = parse_json(oai_resp or "")
+    d = parse_json(dsk_resp or "")
+
+    # Simple ensemble: prefer SUPPORTED/CONTRADICTED if both agree; else pick non-UNVERIFIABLE; else UNVERIFIABLE.
+    def score(st): 
+        return {"SUPPORTED":2,"CONTRADICTED":2,"UNVERIFIABLE":1}.get(st.upper(),1)
+    pick = o if (o.get("status")==d.get("status") and score(o.get("status",""))>=2) else (o if score(o.get("status",""))>score(d.get("status","")) else d)
+    pick["models"] = {"openai": o, "deepseek": d}
+    return pick
+
+async def verify_requirement(requirement_text: str) -> Dict[str, Any]:
+    claims = split_atomic_claims(requirement_text)
+    results = []
+    for c in claims:
+        res = await verify_claim(c)
+        results.append({"claim": c, **res})
+    # roll-up conclusion
+    counts = {"SUPPORTED":0, "CONTRADICTED":0, "UNVERIFIABLE":0}
+    for r in results:
+        counts[r.get("status","UNVERIFIABLE").upper()] = counts.get(r.get("status","UNVERIFIABLE").upper(),0)+1
+    if counts["CONTRADICTED"]>0:
+        overall = "CONTRADICTED"
+    elif counts["SUPPORTED"]>0 and counts["UNVERIFIABLE"]==0:
+        overall = "SUPPORTED"
+    elif counts["SUPPORTED"]>0:
+        overall = "PARTIALLY SUPPORTED"
+    else:
+        overall = "UNVERIFIABLE"
+    return {"claims": results, "summary": counts, "conclusion": overall}