Update app.py

app.py

@@ -5,6 +5,7 @@ import data_preprocess, model, pipeline
 import os
 import hashlib
 import threading
+
 # Gradio UI
 #stop_flag = gr.State(value=False)
 class StopFlag:
@@ -40,9 +41,6 @@ custom_css = """
 }
 """
 
-
-
-
 with gr.Blocks() as interface:
     # with gr.Tab("CURIOUS ABOUT THIS PRODUCT?"):
     #     gr.HTML(value=pricing_html)
@@ -459,6 +457,28 @@ with gr.Blocks() as interface:
                     raise result
         
             return False, None
+
+        from multiprocessing import Pool, cpu_count
+
+        def process_accession_worker(args):
+            acc, stop_flag = args
+            if stop_flag.value:  # Early exit if stop requested before starting
+                return None
+            try:
+                # You can still use run_with_timeout if you want per-accession timeout
+                success, rows = run_with_timeout(
+                    summarize_results,
+                    args=(acc,),
+                    timeout=None,              # or set max seconds per sample if needed
+                    stop_value=stop_flag
+                )
+                if success:
+                    return rows
+                else:
+                    return None
+            except Exception as e:
+                print(f"⚠️ Error processing {acc}: {e}")
+                return None
         
         def threaded_batch_runner(file=None, text="", email=""):
             print("📧 EMAIL RECEIVED:", repr(email))
@@ -608,186 +628,253 @@ with gr.Blocks() as interface:
             #         )
             #         return
     
-                    
-            # Step 2: Loop through accessions
-            for i, acc in enumerate(accessions):
-                if global_stop_flag.value:
-                    log_lines.append(f"🛑 Stopped at {acc} ({i+1}/{total})")
-                    usage_text = ""
+            num_workers = min(cpu_count(), 4)  # HF free tier usually has 2 CPUs
+            pool = Pool(processes=num_workers)
         
-                    if email.strip() and not email_tracked:
-                        print(f"🧪 increment_usage at STOP: {email=} {processed_accessions=}")
-                        usage_count, max_allowed = increment_usage(email, processed_accessions)
-                        email_tracked = True
-                        usage_text = f"**{usage_count}**/{max_allowed} allowed samples used by this email."
-                        #Ten more samples are added first (you now have 60 limited accessions), then wait we will contact you via this email."
-                    else:
-                        usage_text = f"The limited accession is 30. The user has used {processed_accessions}, and only {30 - processed_accessions} left."
+            try:
+                for result in pool.imap_unordered(
+                    process_accession_worker, [(acc, global_stop_flag) for acc in accessions]
+                ):
+                    # if global_stop_flag.value:
+                    #     log_lines.append("🛑 Stop requested — terminating workers...")
+                    #     pool.terminate()
+                    #     break
+                    if global_stop_flag.value:
+                        log_lines.append("🛑 Stopped mid-batch")
+                        usage_text = ""
+                        if email.strip() and not email_tracked:
+                            usage_count, max_allowed = increment_usage(email, processed_accessions)
+                            email_tracked = True
+                            usage_text = f"**{usage_count}**/{max_allowed} allowed samples used."
+                        else:
+                            usage_text = f"The limited accession is 30. The user has used {processed_accessions}, and only {30 - processed_accessions} left."
+                        
+                        pool.terminate()
+                        yield (
+                            make_html_table(all_rows),
+                            gr.update(visible=True),
+                            gr.update(value=output_file_path, visible=bool(output_file_path)),
+                            gr.update(value=usage_text, visible=True),
+                            "🛑 Stopped",
+                            "\n".join(log_lines),
+                            gr.update(visible=False),  # run_button
+                            gr.update(visible=False),  # stop_button
+                            gr.update(visible=True),   # reset_button
+                            gr.update(visible=False),  # raw_text
+                            gr.update(visible=False),  # file_upload
+                            gr.update(value=processed_info, visible=False),
+                            gr.update(visible=True)    # NPS modal
+                        )
+                        return
+        
+                    if result:
+                        all_rows.extend(result)
+                        processed_count += 1
+                        log_lines.append(f"✅ Processed {processed_count}/{total}")
         
-                    # yield (
-                    #     make_html_table(all_rows),
-                    #     gr.update(visible=True),
-                    #     #gr.update(value=output_file_path, visible=True),
-                    #     gr.update(value=output_file_path, visible=bool(output_file_path)),
-                    #     gr.update(value=usage_text, visible=True),
-                    #     "🛑 Stopped",
-                    #     "\n".join(log_lines)
-                    # )
                     yield (
                         make_html_table(all_rows),
-                        gr.update(visible=True),                           # results_group
+                        gr.update(visible=True),  # results_group
                         gr.update(value=output_file_path, visible=bool(output_file_path)),  # download_file
-                        gr.update(value=usage_text, visible=True),         # usage_display
-                        "🛑 Stopped",                                       # "✅ Done" or "🛑 Stopped"
+                        gr.update(value=usage_text, visible=True),  # usage_display
+                        f"⏳ Processed {processed_count}/{total}",
                         "\n".join(log_lines),
-                        gr.update(visible=False),                          # run_button
-                        gr.update(visible=False),                          # stop_button
-                        gr.update(visible=True),                           # reset_button
-                        gr.update(visible=False),                          # raw_text
-                        gr.update(visible=False),                          # file_upload
-                        gr.update(value=processed_info, visible=False),                # processed_info
-                        gr.update(visible=True)                            # NPS modal now visible
+                        gr.update(visible=False),  # run_button
+                        gr.update(visible=True),   # stop_button
+                        gr.update(visible=True),   # reset_button
+                        gr.update(visible=False),  # raw_text
+                        gr.update(visible=False),  # file_upload
+                        gr.update(value="", visible=False),  # processed_info
+                        gr.update(visible=False)   # NPS modal
                     )
+        
+            finally:
+                pool.close()
+                pool.join()
+        
+            if not global_stop_flag.value:
+                log_lines.append("✅ All accessions processed")
+            
+            # # Step 2: Loop through accessions
+            # for i, acc in enumerate(accessions):
+            #     if global_stop_flag.value:
+            #         log_lines.append(f"🛑 Stopped at {acc} ({i+1}/{total})")
+            #         usage_text = ""
+        
+            #         if email.strip() and not email_tracked:
+            #             print(f"🧪 increment_usage at STOP: {email=} {processed_accessions=}")
+            #             usage_count, max_allowed = increment_usage(email, processed_accessions)
+            #             email_tracked = True
+            #             usage_text = f"**{usage_count}**/{max_allowed} allowed samples used by this email."
+            #             #Ten more samples are added first (you now have 60 limited accessions), then wait we will contact you via this email."
+            #         else:
+            #             usage_text = f"The limited accession is 30. The user has used {processed_accessions}, and only {30 - processed_accessions} left."
+        
+            #         # yield (
+            #         #     make_html_table(all_rows),
+            #         #     gr.update(visible=True),
+            #         #     #gr.update(value=output_file_path, visible=True),
+            #         #     gr.update(value=output_file_path, visible=bool(output_file_path)),
+            #         #     gr.update(value=usage_text, visible=True),
+            #         #     "🛑 Stopped",
+            #         #     "\n".join(log_lines)
+            #         # )
+            #         yield (
+            #             make_html_table(all_rows),
+            #             gr.update(visible=True),                           # results_group
+            #             gr.update(value=output_file_path, visible=bool(output_file_path)),  # download_file
+            #             gr.update(value=usage_text, visible=True),         # usage_display
+            #             "🛑 Stopped",                                       # "✅ Done" or "🛑 Stopped"
+            #             "\n".join(log_lines),
+            #             gr.update(visible=False),                          # run_button
+            #             gr.update(visible=False),                          # stop_button
+            #             gr.update(visible=True),                           # reset_button
+            #             gr.update(visible=False),                          # raw_text
+            #             gr.update(visible=False),                          # file_upload
+            #             gr.update(value=processed_info, visible=False),                # processed_info
+            #             gr.update(visible=True)                            # NPS modal now visible
+            #         )
 
-                    return
+            #         return
         
-                log_lines.append(f"[{i+1}/{total}] Processing {acc}")
-                # yield (
-                #     make_html_table(all_rows),
-                #     gr.update(visible=True),
-                #     gr.update(visible=False),
-                #     "",
-                #     "⏳ Processing...",
-                #     "\n".join(log_lines)
-                # )
-                # Hide inputs, show processed_info at start
-                yield (
-                    make_html_table(all_rows),         # output_table
-                    gr.update(visible=True),          # results_group
-                    gr.update(visible=False),          # download_file
-                    "",                                # usage_display
-                    "⏳ Processing...",                 # status
-                    "\n".join(log_lines),              # progress_box
-                    gr.update(visible=False),           # run_button
-                    gr.update(visible=True),           # stop_button
-                    gr.update(visible=True),           # reset_button
-                    gr.update(visible=False),          # hide raw_text
-                    gr.update(visible=False),          # hide file_upload
-                    gr.update(value=processed_info, visible=True),  # processed_info
-                    gr.update(visible=False)           # hide NPS modal at start
-                )
+            #     log_lines.append(f"[{i+1}/{total}] Processing {acc}")
+            #     # yield (
+            #     #     make_html_table(all_rows),
+            #     #     gr.update(visible=True),
+            #     #     gr.update(visible=False),
+            #     #     "",
+            #     #     "⏳ Processing...",
+            #     #     "\n".join(log_lines)
+            #     # )
+            #     # Hide inputs, show processed_info at start
+            #     yield (
+            #         make_html_table(all_rows),         # output_table
+            #         gr.update(visible=True),          # results_group
+            #         gr.update(visible=False),          # download_file
+            #         "",                                # usage_display
+            #         "⏳ Processing...",                 # status
+            #         "\n".join(log_lines),              # progress_box
+            #         gr.update(visible=False),           # run_button
+            #         gr.update(visible=True),           # stop_button
+            #         gr.update(visible=True),           # reset_button
+            #         gr.update(visible=False),          # hide raw_text
+            #         gr.update(visible=False),          # hide file_upload
+            #         gr.update(value=processed_info, visible=True),  # processed_info
+            #         gr.update(visible=False)           # hide NPS modal at start
+            #     )
 
         
-                # try:
-                #     print("📄 Processing accession:", acc)
-                #     rows = summarize_results(acc)
-                #     all_rows.extend(rows)
-                #     processed_accessions += 1  # ✅ only count success
-                #     if email.strip():
-                #         save_to_excel(all_rows, "", "", output_file_path, is_resume=False)
-                #     log_lines.append(f"✅ Processed {acc} ({i+1}/{total})")
-                print("📄 Processing accession:", acc)
-                # --- Before calling summarize_results ---
-                samples_left = total - i  # including current one
-                estimated_seconds_left = samples_left * 100  # your observed average per sample
+            #     # try:
+            #     #     print("📄 Processing accession:", acc)
+            #     #     rows = summarize_results(acc)
+            #     #     all_rows.extend(rows)
+            #     #     processed_accessions += 1  # ✅ only count success
+            #     #     if email.strip():
+            #     #         save_to_excel(all_rows, "", "", output_file_path, is_resume=False)
+            #     #     log_lines.append(f"✅ Processed {acc} ({i+1}/{total})")
+            #     print("📄 Processing accession:", acc)
+            #     # --- Before calling summarize_results ---
+            #     samples_left = total - i  # including current one
+            #     estimated_seconds_left = samples_left * 100  # your observed average per sample
                 
-                log_lines.append(
-                    f"Running... usually ~100s per sample"
-                )
-                log_lines.append(
-                    f"⏳ Estimated time left: ~{estimated_seconds_left} seconds ({samples_left} sample{'s' if samples_left > 1 else ''} remaining)"
-                )
+            #     log_lines.append(
+            #         f"Running... usually ~100s per sample"
+            #     )
+            #     log_lines.append(
+            #         f"⏳ Estimated time left: ~{estimated_seconds_left} seconds ({samples_left} sample{'s' if samples_left > 1 else ''} remaining)"
+            #     )
                 
-                # Yield update to UI before the heavy pipeline call
-                yield (
-                    make_html_table(all_rows),
-                    gr.update(visible=True),          # results_group
-                    gr.update(visible=False),         # download_file
-                    "",                               # usage_display
-                    "⏳ Processing...",                # status
-                    "\n".join(log_lines),             # progress_box
-                    gr.update(visible=False),         # run_button
-                    gr.update(visible=True),          # stop_button
-                    gr.update(visible=True),          # reset_button
-                    gr.update(visible=False),         # raw_text
-                    gr.update(visible=False),         # file_upload
-                    gr.update(value=processed_info, visible=True),  # processed_info
-                    gr.update(visible=False)          # hide NPS modal
-                )
+            #     # Yield update to UI before the heavy pipeline call
+            #     yield (
+            #         make_html_table(all_rows),
+            #         gr.update(visible=True),          # results_group
+            #         gr.update(visible=False),         # download_file
+            #         "",                               # usage_display
+            #         "⏳ Processing...",                # status
+            #         "\n".join(log_lines),             # progress_box
+            #         gr.update(visible=False),         # run_button
+            #         gr.update(visible=True),          # stop_button
+            #         gr.update(visible=True),          # reset_button
+            #         gr.update(visible=False),         # raw_text
+            #         gr.update(visible=False),         # file_upload
+            #         gr.update(value=processed_info, visible=True),  # processed_info
+            #         gr.update(visible=False)          # hide NPS modal
+            #     )
 
-                # Run summarize_results in a separate process with stop flag support
-                success, rows = run_with_timeout(
-                    summarize_results,
-                    args=(acc,),
-                    timeout=None,              # or set max seconds per sample if you want
-                    stop_value=global_stop_flag
-                )
+            #     # Run summarize_results in a separate process with stop flag support
+            #     success, rows = run_with_timeout(
+            #         summarize_results,
+            #         args=(acc,),
+            #         timeout=None,              # or set max seconds per sample if you want
+            #         stop_value=global_stop_flag
+            #     )
                 
-                # If stop was pressed during this accession
-                if not success and global_stop_flag.value:
-                    log_lines.append(f"🛑 Cancelled {acc} before completion")
-                    # yield (
-                    #     make_html_table(all_rows),
-                    #     gr.update(visible=True),
-                    #     gr.update(visible=False),
-                    #     "",
-                    #     "🛑 Stopped",
-                    #     "\n".join(log_lines)
-                    # )
-                    yield (
-                        make_html_table(all_rows),
-                        gr.update(visible=True),                           # results_group
-                        gr.update(value=output_file_path, visible=bool(output_file_path)),  # download_file
-                        gr.update(value=usage_text, visible=True),         # usage_display
-                        "🛑 Stopped",                                       # "✅ Done" or "🛑 Stopped"
-                        "\n".join(log_lines),
-                        gr.update(visible=False),                          # run_button
-                        gr.update(visible=False),                          # stop_button
-                        gr.update(visible=True),                           # reset_button
-                        gr.update(visible=False),                          # raw_text
-                        gr.update(visible=False),                          # file_upload
-                        gr.update(value="", visible=False),                # processed_info
-                        gr.update(visible=True)                            # NPS modal now visible
-                    )
+            #     # If stop was pressed during this accession
+            #     if not success and global_stop_flag.value:
+            #         log_lines.append(f"🛑 Cancelled {acc} before completion")
+            #         # yield (
+            #         #     make_html_table(all_rows),
+            #         #     gr.update(visible=True),
+            #         #     gr.update(visible=False),
+            #         #     "",
+            #         #     "🛑 Stopped",
+            #         #     "\n".join(log_lines)
+            #         # )
+            #         yield (
+            #             make_html_table(all_rows),
+            #             gr.update(visible=True),                           # results_group
+            #             gr.update(value=output_file_path, visible=bool(output_file_path)),  # download_file
+            #             gr.update(value=usage_text, visible=True),         # usage_display
+            #             "🛑 Stopped",                                       # "✅ Done" or "🛑 Stopped"
+            #             "\n".join(log_lines),
+            #             gr.update(visible=False),                          # run_button
+            #             gr.update(visible=False),                          # stop_button
+            #             gr.update(visible=True),                           # reset_button
+            #             gr.update(visible=False),                          # raw_text
+            #             gr.update(visible=False),                          # file_upload
+            #             gr.update(value="", visible=False),                # processed_info
+            #             gr.update(visible=True)                            # NPS modal now visible
+            #         )
 
-                    break  # stop processing entirely
+            #         break  # stop processing entirely
                 
-                # If it finished normally
-                if success and rows:
-                    all_rows.extend(rows)
-                    processed_accessions += 1
-                    if email.strip():
-                        save_to_excel(all_rows, "", "", output_file_path, is_resume=False)
-                    log_lines.append(f"✅ Processed {acc} ({i+1}/{total})")
-                else:
-                    # If it failed due to timeout or other error
-                    if not global_stop_flag.value:
-                        log_lines.append(f"⚠️ Skipped {acc} due to timeout or error")
+            #     # If it finished normally
+            #     if success and rows:
+            #         all_rows.extend(rows)
+            #         processed_accessions += 1
+            #         if email.strip():
+            #             save_to_excel(all_rows, "", "", output_file_path, is_resume=False)
+            #         log_lines.append(f"✅ Processed {acc} ({i+1}/{total})")
+            #     else:
+            #         # If it failed due to timeout or other error
+            #         if not global_stop_flag.value:
+            #             log_lines.append(f"⚠️ Skipped {acc} due to timeout or error")
                 
-                # Always yield updated logs after each attempt
-                # yield (
-                #     make_html_table(all_rows),
-                #     gr.update(visible=True),
-                #     gr.update(visible=False),
-                #     "",
-                #     "⏳ Processing...",
-                #     "\n".join(log_lines)
-                # )
-                yield (
-                    make_html_table(all_rows),         # output_table
-                    gr.update(visible=True),          # results_group
-                    gr.update(visible=False),          # download_file
-                    "",                                # usage_display
-                    "⏳ Processing...",                 # status
-                    "\n".join(log_lines),              # progress_box
-                    gr.update(visible=True),           # run_button
-                    gr.update(visible=True),           # stop_button
-                    gr.update(visible=True),           # reset_button
-                    gr.update(visible=False),          # hide raw_text
-                    gr.update(visible=False),          # hide file_upload
-                    gr.update(value=processed_info, visible=True),  # processed_info
-                    gr.update(visible=False)           # hide NPS modal at start
-                )                
+            #     # Always yield updated logs after each attempt
+            #     # yield (
+            #     #     make_html_table(all_rows),
+            #     #     gr.update(visible=True),
+            #     #     gr.update(visible=False),
+            #     #     "",
+            #     #     "⏳ Processing...",
+            #     #     "\n".join(log_lines)
+            #     # )
+            #     yield (
+            #         make_html_table(all_rows),         # output_table
+            #         gr.update(visible=True),          # results_group
+            #         gr.update(visible=False),          # download_file
+            #         "",                                # usage_display
+            #         "⏳ Processing...",                 # status
+            #         "\n".join(log_lines),              # progress_box
+            #         gr.update(visible=True),           # run_button
+            #         gr.update(visible=True),           # stop_button
+            #         gr.update(visible=True),           # reset_button
+            #         gr.update(visible=False),          # hide raw_text
+            #         gr.update(visible=False),          # hide file_upload
+            #         gr.update(value=processed_info, visible=True),  # processed_info
+            #         gr.update(visible=False)           # hide NPS modal at start
+            #     )
+            
 
                 # except Exception as e:
                 #     log_lines.append(f"❌ Failed to process {acc}: {e}")