Upload folder using huggingface_hub

advanced_rag.py

@@ -81,6 +81,38 @@ jobs = {}  # Stores job status and results
 results_queue = queue.Queue()  # Thread-safe queue for completed jobs
 processing_lock = threading.Lock()  # Prevent simultaneous processing of the same job
 
+# Function to display all jobs as a clickable list
+def get_job_list():
+    job_list_md = "### Submitted Jobs\n\n"
+    
+    if not jobs:
+        return "No jobs found. Submit a query or load files to create jobs."
+    
+    # Sort jobs by start time (newest first)
+    sorted_jobs = sorted(
+        [(job_id, job_info) for job_id, job_info in jobs.items()],
+        key=lambda x: x[1].get("start_time", 0),
+        reverse=True
+    )
+    
+    for job_id, job_info in sorted_jobs:
+        status = job_info.get("status", "unknown")
+        job_type = job_info.get("type", "unknown")
+        query = job_info.get("query", "")
+        start_time = job_info.get("start_time", 0)
+        time_str = datetime.datetime.fromtimestamp(start_time).strftime("%Y-%m-%d %H:%M:%S")
+        
+        # Create a shortened query preview
+        query_preview = query[:30] + "..." if query and len(query) > 30 else query or "N/A"
+        
+        # Create clickable links using Markdown
+        if job_type == "query":
+            job_list_md += f"- [{job_id}](javascript:void) - {time_str} - {status} - Query: {query_preview}\n"
+        else:
+            job_list_md += f"- [{job_id}](javascript:void) - {time_str} - {status} - File Load Job\n"
+    
+    return job_list_md
+
 # Function to process tasks in background
 def process_in_background(job_id: str, function, args):
     try:
@@ -119,13 +151,25 @@ def load_pdfs_async(file_links, model_choice, prompt_template, bm25_weight, temp
     )
 
 # Async version of submit_query_updated
-def submit_query_async(query):
+def submit_query_async(query, model_choice=None):
     if not query:
         return "Please enter a non-empty query", "", "Input tokens: 0", "Output tokens: 0"
     
     if not hasattr(rag_chain, 'elevated_rag_chain') or not rag_chain.raw_data:
         return "Please load files first", "", "Input tokens: 0", "Output tokens: 0"
     
+    # Use the provided model if specified, otherwise use the current model
+    if model_choice and model_choice != "":
+        # Update the model temporarily for this query
+        current_model = rag_chain.llm_choice
+        rag_chain.update_llm_pipeline(
+            model_choice, 
+            rag_chain.temperature, 
+            rag_chain.top_p, 
+            rag_chain.prompt_template, 
+            rag_chain.bm25_weight
+        )
+    
     job_id = str(uuid.uuid4())
     debug_print(f"Starting async job {job_id} for query: {query}")
     
@@ -139,7 +183,8 @@ def submit_query_async(query):
         "status": "processing", 
         "type": "query",
         "start_time": time.time(),
-        "query": query
+        "query": query,
+        "model": rag_chain.llm_choice  # Store which model is being used
     }
     
     return (
@@ -149,11 +194,25 @@ def submit_query_async(query):
         f"Input tokens: {count_tokens(query)}",
         "Output tokens: pending"
     )
+    
+# Function to handle job list clicks
+def job_selected(job_id):
+    if job_id in jobs:
+        return job_id, jobs[job_id].get("query", "No query for this job")
+    return job_id, "Job not found"
+
+# Function to refresh the job list
+def refresh_job_list():
+    return get_job_list()
+
+# Function to sync model dropdown boxes
+def sync_model_dropdown(value):
+    return value    
 
 # Function to check job status
 def check_job_status(job_id):
     if not job_id:
-        return "Please enter a job ID", "", "", ""
+        return "Please enter a job ID", "", "", "", ""
     
     # Process any completed jobs in the queue
     try:
@@ -169,9 +228,10 @@ def check_job_status(job_id):
     
     # Check if the requested job exists
     if job_id not in jobs:
-        return "Job not found. Please check the ID and try again.", "", "", ""
+        return "Job not found. Please check the ID and try again.", "", "", "", ""
     
     job = jobs[job_id]
+    job_query = job.get("query", "No query available for this job")
     
     # If job is still processing
     if job["status"] == "processing":
@@ -183,7 +243,9 @@ def check_job_status(job_id):
                 f"Files are still being processed (elapsed: {elapsed_time:.1f}s).\n\n"
                 f"Try checking again in a few seconds.",
                 f"Job ID: {job_id}",
-                f"Status: Processing"
+                f"Status: Processing",
+                "",
+                job_query
             )
         else:  # query job
             return (
@@ -191,7 +253,8 @@ def check_job_status(job_id):
                 f"Try checking again in a few seconds.",
                 f"Job ID: {job_id}",
                 f"Input tokens: {count_tokens(job.get('query', ''))}",
-                "Output tokens: pending"
+                "Output tokens: pending",
+                job_query
             )
     
     # If job is completed
@@ -203,18 +266,21 @@ def check_job_status(job_id):
             return (
                 f"{result[0]}\n\nProcessing time: {processing_time:.1f}s",
                 result[1],
-                result[2] 
+                result[2],
+                "",
+                job_query
             )
         else:  # query job
             return (
                 f"{result[0]}\n\nProcessing time: {processing_time:.1f}s",
                 result[1],
                 result[2],
-                result[3]
+                result[3],
+                job_query
             )
     
     # Fallback for unknown status
-    return f"Job status: {job['status']}", "", "", ""
+    return f"Job status: {job['status']}", "", "", "", job_query
 
 # Function to clean up old jobs
 def cleanup_old_jobs():
@@ -352,7 +418,7 @@ class ElevatedRagChain:
                                 model=repo_id,
                                 temperature=self.temperature,
                                 top_p=self.top_p,
-                                max_new_tokens=8192  # Reduced token count for speed
+                                max_new_tokens=512  # Reduced token count for speed
                             )
                             return response
                         except Exception as e:
@@ -744,7 +810,30 @@ textarea {
 """
 
 # Update the Gradio interface to include job status checking
-with gr.Blocks(css=custom_css) as app:
+with gr.Blocks(css=custom_css, js="""
+document.addEventListener('DOMContentLoaded', function() {
+    // Add event listener for job list clicks
+    const jobListInterval = setInterval(() => {
+        const jobLinks = document.querySelectorAll('.job-list-container a');
+        if (jobLinks.length > 0) {
+            jobLinks.forEach(link => {
+                link.addEventListener('click', function(e) {
+                    e.preventDefault();
+                    const jobId = this.textContent.split(' ')[0];
+                    // Find the job ID input textbox and set its value
+                    const jobIdInput = document.querySelector('.job-id-input input');
+                    if (jobIdInput) {
+                        jobIdInput.value = jobId;
+                        // Trigger the input event to update Gradio's state
+                        jobIdInput.dispatchEvent(new Event('input', { bubbles: true }));
+                    }
+                });
+            });
+            clearInterval(jobListInterval);
+        }
+    }, 500);
+});
+""") as app:
     gr.Markdown('''# PhiRAG - Async Version  
 **PhiRAG** Query Your Data with Advanced RAG Techniques
 
@@ -851,31 +940,45 @@ https://www.gutenberg.org/ebooks/8438.txt.utf-8
                 input_tokens = gr.Markdown("Input tokens: 0")
                 output_tokens = gr.Markdown("Output tokens: 0")
         
-        with gr.TabItem("Check Job Status"):
-            with gr.Row():
-                job_id_input = gr.Textbox(
-                    label="Enter Job ID",
-                    placeholder="Paste the Job ID here",
-                    lines=1
-                )
-                check_button = gr.Button("Check Status")
-                cleanup_button = gr.Button("Cleanup Old Jobs")
-            
-            with gr.Row():
-                status_response = gr.Textbox(
-                    label="Job Result",
-                    placeholder="Job result will appear here",
-                    lines=6
-                )
-                status_context = gr.Textbox(
-                    label="Context Information",
-                    placeholder="Context information will appear here",
-                    lines=6
-                )
-            
-            with gr.Row():
-                status_tokens1 = gr.Markdown("")
-                status_tokens2 = gr.Markdown("")
+            with gr.TabItem("Check Job Status"):
+                with gr.Row():
+                    with gr.Column(scale=1):
+                        job_list = gr.Markdown(
+                            value="No jobs yet",
+                            label="Job List (Click to select)"
+                        )
+                        refresh_button = gr.Button("Refresh Job List")
+                    
+                    with gr.Column(scale=2):
+                        job_id_input = gr.Textbox(
+                            label="Job ID",
+                            placeholder="Job ID will appear here when selected from the list",
+                            lines=1
+                        )
+                        job_query_display = gr.Textbox(
+                            label="Job Query",
+                            placeholder="The query associated with this job will appear here",
+                            lines=2,
+                            interactive=False
+                        )
+                        check_button = gr.Button("Check Status")
+                        cleanup_button = gr.Button("Cleanup Old Jobs")
+                
+                with gr.Row():
+                    status_response = gr.Textbox(
+                        label="Job Result",
+                        placeholder="Job result will appear here",
+                        lines=6
+                    )
+                    status_context = gr.Textbox(
+                        label="Context Information",
+                        placeholder="Context information will appear here",
+                        lines=6
+                    )
+                
+                with gr.Row():
+                    status_tokens1 = gr.Markdown("")
+                    status_tokens2 = gr.Markdown("")
         
         with gr.TabItem("App Management"):
             with gr.Row():
@@ -903,35 +1006,62 @@ https://www.gutenberg.org/ebooks/8438.txt.utf-8
         inputs=[pdf_input, model_dropdown, prompt_input, bm25_weight_slider, temperature_slider, top_p_slider],
         outputs=[load_response, load_context, model_output]
     )
-    
+
     submit_button.click(
         submit_query_async, 
-        inputs=[query_input],
+        inputs=[query_input, query_model_dropdown],
         outputs=[query_response, query_context, input_tokens, output_tokens]
     )
-    
+
     check_button.click(
         check_job_status,
         inputs=[job_id_input],
-        outputs=[status_response, status_context, status_tokens1, status_tokens2]
+        outputs=[status_response, status_context, status_tokens1, status_tokens2, job_query_display]
     )
-    
+
+    refresh_button.click(
+        refresh_job_list,
+        inputs=[],
+        outputs=[job_list]
+    )
+
+    # Connect the job list selection event (this is handled by JavaScript)
+    job_id_input.change(
+        job_selected,
+        inputs=[job_id_input],
+        outputs=[job_id_input, job_query_display]
+    )
+
     cleanup_button.click(
         cleanup_old_jobs,
         inputs=[],
         outputs=[status_response, status_context, status_tokens1]
     )
-    
+
     reset_button.click(
         reset_app_updated, 
         inputs=[], 
         outputs=[reset_response, reset_context, reset_model]
     )
-    
+
     model_dropdown.change(
-        fn=update_model,
+        fn=sync_model_dropdown,
         inputs=model_dropdown,
-        outputs=model_output
+        outputs=query_model_dropdown
+    )
+
+    # Also sync in the other direction
+    query_model_dropdown.change(
+        fn=sync_model_dropdown,
+        inputs=query_model_dropdown,
+        outputs=model_dropdown
+    )
+
+    # Add an event to refresh the job list on page load
+    app.load(
+        fn=refresh_job_list,
+        inputs=None,
+        outputs=job_list
     )
     
 if __name__ == "__main__":