""" FilterLM — Gradio UI for PDF criteria search (no LLMs). Uses the same quote/text matching as pdf_criteria_search_gui_v2.py. Background jobs, job list, Refresh, and auto-refresh mirror the execution pattern in psyllm.py. """ from __future__ import annotations import datetime import html as html_module import os import queue import tempfile import threading import time import uuid from typing import Any, Dict, List, Tuple import gradio as gr from search_keywords import CATEGORY_HEADINGS, CATEGORY_STYLES, KEY_TO_CATEGORY from pdf_criteria_search_gui_v2 import ( QuoteSearchEngine, best_highlight_phrase_for_group, export_docx, parse_terms_text, phrases_for_group_key, terms_csv_path, ensure_terms_csv, write_terms_to_csv, load_terms_from_csv, get_pdf_title_and_first_author, ) # ----------------------------------------------------------------------------- # Job infrastructure (aligned with psyllm.py batch/async pattern) # ----------------------------------------------------------------------------- jobs: Dict[str, Dict[str, Any]] = {} results_queue: queue.Queue = queue.Queue() last_job_id: str | None = None def debug_print(message: str) -> None: print(f"[{datetime.datetime.now().isoformat()}] {message}", flush=True) def flatten_upload_inputs(files: Any) -> List[Any]: """Gradio may return a single path, a list, nested lists, or a numpy array.""" if files is None: return [] try: import numpy as np if isinstance(files, np.ndarray): files = files.tolist() except ImportError: pass if isinstance(files, (list, tuple)): acc: List[Any] = [] for item in files: acc.extend(flatten_upload_inputs(item)) return acc return [files] def _upload_item_to_path(item: Any) -> str | None: if item is None: return None path: str | None = None if isinstance(item, str): path = item elif isinstance(item, dict): path = item.get("path") or item.get("name") elif hasattr(item, "name"): n = getattr(item, "name") if isinstance(n, str): path = n if path is None: try: cand = os.fspath(item) if isinstance(cand, str): path = cand except Exception: path = None if path and isinstance(path, str) and os.path.isfile(path) and path.lower().endswith(".pdf"): return os.path.abspath(path) return None def normalize_upload_files(files: Any) -> List[str]: """Resolve every uploaded PDF path (supports multi-select drag-and-drop).""" out: List[str] = [] for item in flatten_upload_inputs(files): p = _upload_item_to_path(item) if p: out.append(p) seen: set[str] = set() uniq: List[str] = [] for p in out: if p not in seen: seen.add(p) uniq.append(p) return uniq def terms_keys_html(raw: str) -> str: """Colored HTML preview of keys + category bands (same grouping as Tk GUI).""" keys = parse_terms_text(raw) if not keys: return '
No keys yet — add one group id per line (# = comment).
' parts: List[str] = ['No PDFs loaded — drag files here or use a folder path below.
' ) rows = [] for i, p in enumerate(paths, start=1): base = html_module.escape(os.path.basename(p)) full = html_module.escape(p) rows.append(f"{len(paths)} PDF(s) ready to search.
' f'| # | File | Path |
|---|
(no matches)
') continue sorted_blocks = sorted(blocks, key=lambda b: os.path.basename(b["pdf_path"]).lower()) for block in sorted_blocks: fp = block["pdf_path"] base = os.path.basename(fp) if fp not in pdf_title_cache: pdf_title_cache[fp] = get_pdf_title_and_first_author(fp) title, fa = pdf_title_cache[fp] chunks.append( "%s — %s — %s
" % ( html_module.escape(base), html_module.escape(title), html_module.escape(fa), ) ) occs = block.get("occurrences", []) if grouped_pages: pages = ", ".join(str(o["page"]) for o in occs) apx = "yes" if any(o.get("approximate") for o in occs) else "no" chunks.append( "Pages: %s | approximate: %s
" % (html_module.escape(pages), html_module.escape(apx)) ) for o in occs: ex = (o.get("excerpt") or "").strip() if ex: chunks.append( 'p.%s: %s
' % ( html_module.escape(str(o["page"])), excerpt_html_highlighted(term, ex), ) ) else: for o in occs: apx = " (approx.)" if o.get("approximate") else "" ex = (o.get("excerpt") or "").strip() chunks.append( 'p.%s%s: %s
' % ( html_module.escape(str(o["page"])), html_module.escape(apx), excerpt_html_highlighted(term, ex), ) ) chunks.append("") chunks.append("") chunks.append("Searching PDFs… colored results appear here when finished.
' if not (job_id or "").strip(): return ("Enter a job ID or click one in the list.", "", "", None, "") jid = job_id.strip() if jid not in jobs: return ( "Job not found. Refresh the list or copy the ID from the submission message.", "", "", None, "", ) job = jobs[jid] paths = job.get("pdf_paths") or [] summary = ( f"Source: {job.get('folder', '')}\n" f"PDFs: {len(paths)}\n" f"Terms: {job.get('terms_count', '')}" ) if job["status"] == "processing": elapsed = time.time() - job["start_time"] header = generate_job_status_report(jid, job) body = f"\n\nStill running… ({elapsed:.1f}s elapsed)." return header + body, waiting_html, "", None, summary if job["status"] == "completed": raw = job.get("result") if isinstance(raw, tuple) and raw and raw[0] == "error": err = raw[1] if len(raw) > 1 else "unknown error" err_html = f"{html_module.escape(str(err))}"
return (
generate_job_status_report(jid, job) + f"\n\n**Error:** {err}",
err_html,
"",
None,
summary,
)
if isinstance(raw, tuple) and len(raw) >= 6 and raw[0] == "ok":
_ok, _plain, report_html, docx_path, stats, _folder = raw[:6]
header = generate_job_status_report(jid, job)
dp = docx_path if docx_path and os.path.isfile(docx_path) else None
return header, report_html, _stats_as_md(stats), dp, summary
fallback = f"{html_module.escape(str(raw))}"
return generate_job_status_report(jid, job), fallback, "", None, summary
return generate_job_status_report(jid, job), "", "", None, ""
def cleanup_old_jobs() -> Tuple[str, str, str]:
current_time = time.time()
to_delete: List[str] = []
for job_id, job in jobs.items():
if job["status"] == "completed" and (current_time - job.get("end_time", 0)) > 86400:
to_delete.append(job_id)
elif job["status"] == "processing" and (current_time - job.get("start_time", 0)) > 172800:
to_delete.append(job_id)
for job_id in to_delete:
del jobs[job_id]
debug_print(f"Cleaned up {len(to_delete)} job(s); {len(jobs)} remaining.")
return f"Removed {len(to_delete)} old job(s).", "", ""
def periodic_update_fixed(is_checked: bool):
"""Refresh job list + last job status every ~2s when auto-refresh is on."""
jl = refresh_job_list()
if not is_checked:
return jl, gr.skip(), gr.skip(), gr.skip(), gr.skip(), gr.skip()
if last_job_id:
return (jl,) + check_job_status(last_job_id)
return (
jl,
"No job ID yet — submit a search first.",
'Submit a search first.
', "", None, "", ) def save_terms_to_disk(raw: str) -> Tuple[str, str]: terms = parse_terms_text(raw) preview = terms_keys_html(raw) if not terms: return "Nothing to save (empty terms).", preview path = terms_csv_path() ensure_terms_csv(path) try: write_terms_to_csv(path, terms) return f"Saved {len(terms)} keys to `{path}`.", preview except OSError as e: return f"Save failed: {e}", preview def load_terms_for_ui() -> str: path = terms_csv_path() ensure_terms_csv(path) keys = load_terms_from_csv(path) return "\n".join(keys) # ----------------------------------------------------------------------------- # Gradio layout (tabs/jobs pattern similar to psyllm.py) # ----------------------------------------------------------------------------- custom_css = """ textarea { overflow-y: scroll !important; max-height: 220px; } .job-list-container a { cursor: pointer; } .muted { color: #616161; font-style: italic; } .cat-head { background: #eceff1; color: #37474f; font-weight: 600; padding: 6px 10px; margin-top: 10px; border-radius: 4px; font-size: 13px; } .term-chip { display: inline-block; padding: 3px 10px; margin: 3px 0; border-radius: 4px; font-family: Consolas, monospace; font-size: 13px; } .pdf-loaded-table { width: 100%; border-collapse: collapse; font-size: 13px; margin-top: 8px; } .pdf-loaded-table th, .pdf-loaded-table td { border: 1px solid #cfd8dc; padding: 6px 8px; text-align: left; } .pdf-loaded-table thead { background: #eceff1; } .path-cell { word-break: break-all; font-size: 12px; color: #424242; } .results-report { font-family: system-ui, Segoe UI, sans-serif; line-height: 1.45; font-size: 14px; } .result-term-h3 { padding: 8px 12px; margin: 14px 0 8px 0; border-radius: 4px; font-size: 1.05rem; } .results-report .indent { margin-left: 12px; color: #455a64; } .results-report .excerpt { margin: 4px 0 4px 16px; padding-left: 8px; border-left: 3px solid #b0bec5; } .kw-hit { background: #006400; color: #ffffff; padding: 0 3px; border-radius: 2px; } .error-pre { background: #ffebee; padding: 10px; border-radius: 4px; overflow-x: auto; } """ JS_CLICK_JOB_ID = r""" document.addEventListener('DOMContentLoaded', function() { const jobListInterval = setInterval(() => { const jobLinks = document.querySelectorAll('.job-list-container a'); if (jobLinks.length > 0) { jobLinks.forEach(link => { link.addEventListener('click', function(e) { e.preventDefault(); const jobId = this.textContent.trim().split(/\s/)[0]; const jobIdInput = document.querySelector('.job-id-input input'); if (jobIdInput) { jobIdInput.value = jobId; jobIdInput.dispatchEvent(new Event('input', { bubbles: true })); } }); }); clearInterval(jobListInterval); } }, 500); }); """ with gr.Blocks(css=custom_css, js=JS_CLICK_JOB_ID) as app: gr.Markdown( """# DOX — parallel search using multiple keywords in multiple PDFs Searches in parallel for keywords (organized in groups) against multiple PDFs using the same normalization / fuzzy logic. No LLMs are used. Each line is a **group id**. Jobs run **in the background** — use **Refresh** / **Enable Auto Refresh** on the job list below. """ ) with gr.Tabs(): with gr.TabItem("PDF Criteria Search"): submit_btn = gr.Button("Submit PDF search", variant="primary") submit_msg = gr.Textbox(label="Submission log", lines=4) gr.Markdown( "**PDF sources:** Drag **multiple** PDFs below (listed immediately), **or** enter a folder path. " "If both are set, **uploaded files are searched**. " "In the file dialog, use **Ctrl+click** / **Shift+click** to select several PDFs." ) with gr.Row(): with gr.Column(scale=1): pdf_upload = gr.File( label="PDF files (multi-select drag-and-drop or browse)", file_count="multiple", file_types=[".pdf"], ) loaded_files_html = gr.HTML(value=loaded_pdfs_html(None)) with gr.Column(scale=1): folder_input = gr.Textbox( label="Folder with PDFs (optional if uploading)", placeholder=r"C:\path\to\pdfs", lines=1, ) terms_box = gr.Textbox( label="Group keys (one per line; # starts a comment)", value=load_terms_for_ui(), lines=14, ) terms_html_preview = gr.HTML( label="Keys by category (colors)", value=terms_keys_html(load_terms_for_ui()), ) with gr.Row(): group_toggle = gr.Checkbox(label="GROUP page hits per PDF", value=True) save_csv_btn = gr.Button("Save keys to CSV") gr.Markdown("### Jobs & results") with gr.Row(): with gr.Column(scale=1): job_list = gr.Markdown( value="No jobs yet", label="Job list (click an ID)", elem_classes=["job-list-container"], ) refresh_button = gr.Button("Refresh Job List") auto_refresh_checkbox = gr.Checkbox( label="Enable Auto Refresh", value=False, ) with gr.Column(scale=2): job_id_input = gr.Textbox( label="Job ID", placeholder="Paste ID or click from the list", lines=1, elem_classes=["job-id-input"], ) job_summary = gr.Textbox( label="Job summary", lines=3, interactive=False, ) check_button = gr.Button("Check Status") cleanup_button = gr.Button("Cleanup old jobs") status_header = gr.Markdown(label="Status") results_html = gr.HTML(label="Search results (category headings + green keyword span)") stats_box = gr.Markdown(label="Stats") docx_file = gr.File(label="Exported DOCX (when ready)", interactive=False) pdf_upload.change(loaded_pdfs_html, inputs=[pdf_upload], outputs=[loaded_files_html]) terms_box.change(terms_keys_html, inputs=[terms_box], outputs=[terms_html_preview]) save_csv_btn.click(save_terms_to_disk, inputs=[terms_box], outputs=[submit_msg, terms_html_preview]) submit_btn.click( submit_pdf_search, inputs=[folder_input, pdf_upload, terms_box, group_toggle], outputs=[submit_msg, job_id_input, job_summary, job_list], ) refresh_button.click(refresh_job_list, outputs=[job_list]) check_button.click( check_job_status, inputs=[job_id_input], outputs=[ status_header, results_html, stats_box, docx_file, job_summary, ], ) job_id_input.change(job_selected, inputs=[job_id_input], outputs=[job_id_input, job_summary]) cleanup_button.click(cleanup_old_jobs, outputs=[status_header, results_html, stats_box]) auto_refresh_checkbox.change( fn=lambda chk: periodic_update_fixed(chk), inputs=[auto_refresh_checkbox], outputs=[ job_list, status_header, results_html, stats_box, docx_file, job_summary, ], every=2, ) with gr.TabItem("App Management"): gr.Markdown( "**Cleanup old jobs** removes completed jobs older than 24h " "and stale processing entries (48h). Running searches are unaffected unless stale." ) def load_app(): drain_results_queue() keys_txt = load_terms_for_ui() return get_job_list(), terms_keys_html(keys_txt), loaded_pdfs_html(None) app.load(fn=load_app, outputs=[job_list, terms_html_preview, loaded_files_html]) if __name__ == "__main__": debug_print("Launching FilterLM (Gradio).") app.queue().launch(share=False)