Update app.py

app.py

@@ -5,8 +5,6 @@ import data_preprocess, model, pipeline
 import os
 import hashlib
 import threading
-import multiprocessing as mp
-mp.set_start_method("spawn", force=True)  # ensures workers aren't daemonic
 
 # Gradio UI
 #stop_flag = gr.State(value=False)
@@ -460,28 +458,6 @@ with gr.Blocks() as interface:
         
             return False, None
 
-        from multiprocessing import Pool, cpu_count
-
-        def process_accession_worker(args):
-            acc, stop_flag = args
-            if stop_flag.value:  # Early exit if stop requested before starting
-                return None
-            try:
-                # You can still use run_with_timeout if you want per-accession timeout
-                success, rows = run_with_timeout(
-                    mtdna_backend.summarize_results,
-                    args=(acc,),
-                    timeout=None,              # or set max seconds per sample if needed
-                    stop_value=stop_flag
-                )
-                if success:
-                    return rows
-                else:
-                    return None
-            except Exception as e:
-                print(f"⚠️ Error processing {acc}: {e}")
-                return None
-        from concurrent.futures import ProcessPoolExecutor, as_completed
         def threaded_batch_runner(file=None, text="", email=""):
             print("📧 EMAIL RECEIVED:", repr(email))
             import tempfile
@@ -629,271 +605,207 @@ with gr.Blocks() as interface:
             #             "\n".join(log_lines)
             #         )
             #         return
-    
-            from multiprocessing import get_context
-
-            num_workers = min(cpu_count(), 4)  # HF free tier usually has 2 CPUs
-            ctx = get_context("spawn")  # ensures workers are not daemonic
-            pool = ctx.Pool(processes=num_workers)
-
-        
-            try:
-                for result in pool.imap_unordered(
-                    process_accession_worker, [(acc, global_stop_flag) for acc in accessions]
-                ):
-                    # if global_stop_flag.value:
-                    #     log_lines.append("🛑 Stop requested — terminating workers...")
-                    #     pool.terminate()
-                    #     break
+            
+            # Step 2: Loop through accessions
+            for i, acc in enumerate(accessions):
+                try:
                     if global_stop_flag.value:
-                        log_lines.append("🛑 Stopped mid-batch")
+                        log_lines.append(f"🛑 Stopped at {acc} ({i+1}/{total})")
                         usage_text = ""
+            
                         if email.strip() and not email_tracked:
+                            print(f"🧪 increment_usage at STOP: {email=} {processed_accessions=}")
                             usage_count, max_allowed = increment_usage(email, processed_accessions)
                             email_tracked = True
-                            usage_text = f"**{usage_count}**/{max_allowed} allowed samples used."
+                            usage_text = f"**{usage_count}**/{max_allowed} allowed samples used by this email."
+                            #Ten more samples are added first (you now have 60 limited accessions), then wait we will contact you via this email."
                         else:
                             usage_text = f"The limited accession is 30. The user has used {processed_accessions}, and only {30 - processed_accessions} left."
-                        
-                        pool.terminate()
+            
+                        # yield (
+                        #     make_html_table(all_rows),
+                        #     gr.update(visible=True),
+                        #     #gr.update(value=output_file_path, visible=True),
+                        #     gr.update(value=output_file_path, visible=bool(output_file_path)),
+                        #     gr.update(value=usage_text, visible=True),
+                        #     "🛑 Stopped",
+                        #     "\n".join(log_lines)
+                        # )
                         yield (
                             make_html_table(all_rows),
-                            gr.update(visible=True),
-                            gr.update(value=output_file_path, visible=bool(output_file_path)),
-                            gr.update(value=usage_text, visible=True),
-                            "🛑 Stopped",
+                            gr.update(visible=True),                           # results_group
+                            gr.update(value=output_file_path, visible=bool(output_file_path)),  # download_file
+                            gr.update(value=usage_text, visible=True),         # usage_display
+                            "🛑 Stopped",                                       # "✅ Done" or "🛑 Stopped"
                             "\n".join(log_lines),
-                            gr.update(visible=False),  # run_button
-                            gr.update(visible=False),  # stop_button
-                            gr.update(visible=True),   # reset_button
-                            gr.update(visible=False),  # raw_text
-                            gr.update(visible=False),  # file_upload
-                            gr.update(value=processed_info, visible=True),
-                            gr.update(visible=True)    # NPS modal
+                            gr.update(visible=False),                          # run_button
+                            gr.update(visible=False),                          # stop_button
+                            gr.update(visible=True),                           # reset_button
+                            gr.update(visible=False),                          # raw_text
+                            gr.update(visible=False),                          # file_upload
+                            gr.update(value=processed_info, visible=False),                # processed_info
+                            gr.update(visible=True)                            # NPS modal now visible
                         )
+    
                         return
-        
-                    if result:
-                        all_rows.extend(result)
-                        processed_accessions += 1
-                        log_lines.append(f"✅ Processed {processed_accessions}/{total}")
-        
+            
+                    log_lines.append(f"[{i+1}/{total}] Processing {acc}")
+                    # yield (
+                    #     make_html_table(all_rows),
+                    #     gr.update(visible=True),
+                    #     gr.update(visible=False),
+                    #     "",
+                    #     "⏳ Processing...",
+                    #     "\n".join(log_lines)
+                    # )
+                    # Hide inputs, show processed_info at start
+                    yield (
+                        make_html_table(all_rows),         # output_table
+                        gr.update(visible=True),          # results_group
+                        gr.update(visible=False),          # download_file
+                        "",                                # usage_display
+                        "⏳ Processing...",                 # status
+                        "\n".join(log_lines),              # progress_box
+                        gr.update(visible=False),           # run_button
+                        gr.update(visible=True),           # stop_button
+                        gr.update(visible=True),           # reset_button
+                        gr.update(visible=False),          # hide raw_text
+                        gr.update(visible=False),          # hide file_upload
+                        gr.update(value=processed_info, visible=True),  # processed_info
+                        gr.update(visible=False)           # hide NPS modal at start
+                    )
+    
+            
+                    # try:
+                    #     print("📄 Processing accession:", acc)
+                    #     rows = summarize_results(acc)
+                    #     all_rows.extend(rows)
+                    #     processed_accessions += 1  # ✅ only count success
+                    #     if email.strip():
+                    #         save_to_excel(all_rows, "", "", output_file_path, is_resume=False)
+                    #     log_lines.append(f"✅ Processed {acc} ({i+1}/{total})")
+                    print("📄 Processing accession:", acc)
+                    # --- Before calling summarize_results ---
+                    samples_left = total - i  # including current one
+                    estimated_seconds_left = samples_left * 100  # your observed average per sample
+                    
+                    log_lines.append(
+                        f"Running... usually ~100s per sample"
+                    )
+                    log_lines.append(
+                        f"⏳ Estimated time left: ~{estimated_seconds_left} seconds ({samples_left} sample{'s' if samples_left > 1 else ''} remaining)"
+                    )
+                    
+                    # Yield update to UI before the heavy pipeline call
                     yield (
                         make_html_table(all_rows),
-                        gr.update(visible=True),  # results_group
-                        gr.update(value=output_file_path, visible=bool(output_file_path)),  # download_file
-                        gr.update(value=usage_text, visible=True),  # usage_display
-                        f"⏳ Processed {processed_accessions}/{total}",
-                        "\n".join(log_lines),
-                        gr.update(visible=False),  # run_button
-                        gr.update(visible=True),   # stop_button
-                        gr.update(visible=True),   # reset_button
-                        gr.update(visible=False),  # raw_text
-                        gr.update(visible=False),  # file_upload
+                        gr.update(visible=True),          # results_group
+                        gr.update(visible=False),         # download_file
+                        "",                               # usage_display
+                        "⏳ Processing...",                # status
+                        "\n".join(log_lines),             # progress_box
+                        gr.update(visible=False),         # run_button
+                        gr.update(visible=True),          # stop_button
+                        gr.update(visible=True),          # reset_button
+                        gr.update(visible=False),         # raw_text
+                        gr.update(visible=False),         # file_upload
                         gr.update(value=processed_info, visible=True),  # processed_info
-                        gr.update(visible=False)   # NPS modal
+                        gr.update(visible=False)          # hide NPS modal
+                    )
+    
+                    # Run summarize_results in a separate process with stop flag support
+                    success, rows = run_with_timeout(
+                        summarize_results,
+                        args=(acc,),
+                        timeout=None,              # or set max seconds per sample if you want
+                        stop_value=global_stop_flag
+                    )
+                    
+                    # If stop was pressed during this accession
+                    if not success and global_stop_flag.value:
+                        log_lines.append(f"🛑 Cancelled {acc} before completion")
+                        # yield (
+                        #     make_html_table(all_rows),
+                        #     gr.update(visible=True),
+                        #     gr.update(visible=False),
+                        #     "",
+                        #     "🛑 Stopped",
+                        #     "\n".join(log_lines)
+                        # )
+                        yield (
+                            make_html_table(all_rows),
+                            gr.update(visible=True),                           # results_group
+                            gr.update(value=output_file_path, visible=bool(output_file_path)),  # download_file
+                            gr.update(value=usage_text, visible=True),         # usage_display
+                            "🛑 Stopped",                                       # "✅ Done" or "🛑 Stopped"
+                            "\n".join(log_lines),
+                            gr.update(visible=False),                          # run_button
+                            gr.update(visible=False),                          # stop_button
+                            gr.update(visible=True),                           # reset_button
+                            gr.update(visible=False),                          # raw_text
+                            gr.update(visible=False),                          # file_upload
+                            gr.update(value="", visible=False),                # processed_info
+                            gr.update(visible=True)                            # NPS modal now visible
+                        )
+    
+                        break  # stop processing entirely
+                    
+                    # If it finished normally
+                    if success and rows:
+                        all_rows.extend(rows)
+                        processed_accessions += 1
+                        if email.strip():
+                            save_to_excel(all_rows, "", "", output_file_path, is_resume=False)
+                        log_lines.append(f"✅ Processed {acc} ({i+1}/{total})")
+                    else:
+                        # If it failed due to timeout or other error
+                        if not global_stop_flag.value:
+                            log_lines.append(f"⚠️ Skipped {acc} due to timeout or error")
+                    
+                    # Always yield updated logs after each attempt
+                    # yield (
+                    #     make_html_table(all_rows),
+                    #     gr.update(visible=True),
+                    #     gr.update(visible=False),
+                    #     "",
+                    #     "⏳ Processing...",
+                    #     "\n".join(log_lines)
+                    # )
+                    yield (
+                        make_html_table(all_rows),         # output_table
+                        gr.update(visible=True),          # results_group
+                        gr.update(visible=False),          # download_file
+                        "",                                # usage_display
+                        "⏳ Processing...",                 # status
+                        "\n".join(log_lines),              # progress_box
+                        gr.update(visible=True),           # run_button
+                        gr.update(visible=True),           # stop_button
+                        gr.update(visible=True),           # reset_button
+                        gr.update(visible=False),          # hide raw_text
+                        gr.update(visible=False),          # hide file_upload
+                        gr.update(value=processed_info, visible=True),  # processed_info
+                        gr.update(visible=False)           # hide NPS modal at start
                     )
-        
-            finally:
-                pool.close()
-                pool.join()
-        
-            if not global_stop_flag.value:
-                log_lines.append("✅ All accessions processed")
             
-            # # Step 2: Loop through accessions
-            # for i, acc in enumerate(accessions):
-            #     if global_stop_flag.value:
-            #         log_lines.append(f"🛑 Stopped at {acc} ({i+1}/{total})")
-            #         usage_text = ""
-        
-            #         if email.strip() and not email_tracked:
-            #             print(f"🧪 increment_usage at STOP: {email=} {processed_accessions=}")
-            #             usage_count, max_allowed = increment_usage(email, processed_accessions)
-            #             email_tracked = True
-            #             usage_text = f"**{usage_count}**/{max_allowed} allowed samples used by this email."
-            #             #Ten more samples are added first (you now have 60 limited accessions), then wait we will contact you via this email."
-            #         else:
-            #             usage_text = f"The limited accession is 30. The user has used {processed_accessions}, and only {30 - processed_accessions} left."
-        
-            #         # yield (
-            #         #     make_html_table(all_rows),
-            #         #     gr.update(visible=True),
-            #         #     #gr.update(value=output_file_path, visible=True),
-            #         #     gr.update(value=output_file_path, visible=bool(output_file_path)),
-            #         #     gr.update(value=usage_text, visible=True),
-            #         #     "🛑 Stopped",
-            #         #     "\n".join(log_lines)
-            #         # )
-            #         yield (
-            #             make_html_table(all_rows),
-            #             gr.update(visible=True),                           # results_group
-            #             gr.update(value=output_file_path, visible=bool(output_file_path)),  # download_file
-            #             gr.update(value=usage_text, visible=True),         # usage_display
-            #             "🛑 Stopped",                                       # "✅ Done" or "🛑 Stopped"
-            #             "\n".join(log_lines),
-            #             gr.update(visible=False),                          # run_button
-            #             gr.update(visible=False),                          # stop_button
-            #             gr.update(visible=True),                           # reset_button
-            #             gr.update(visible=False),                          # raw_text
-            #             gr.update(visible=False),                          # file_upload
-            #             gr.update(value=processed_info, visible=False),                # processed_info
-            #             gr.update(visible=True)                            # NPS modal now visible
-            #         )
-
-            #         return
-        
-            #     log_lines.append(f"[{i+1}/{total}] Processing {acc}")
-            #     # yield (
-            #     #     make_html_table(all_rows),
-            #     #     gr.update(visible=True),
-            #     #     gr.update(visible=False),
-            #     #     "",
-            #     #     "⏳ Processing...",
-            #     #     "\n".join(log_lines)
-            #     # )
-            #     # Hide inputs, show processed_info at start
-            #     yield (
-            #         make_html_table(all_rows),         # output_table
-            #         gr.update(visible=True),          # results_group
-            #         gr.update(visible=False),          # download_file
-            #         "",                                # usage_display
-            #         "⏳ Processing...",                 # status
-            #         "\n".join(log_lines),              # progress_box
-            #         gr.update(visible=False),           # run_button
-            #         gr.update(visible=True),           # stop_button
-            #         gr.update(visible=True),           # reset_button
-            #         gr.update(visible=False),          # hide raw_text
-            #         gr.update(visible=False),          # hide file_upload
-            #         gr.update(value=processed_info, visible=True),  # processed_info
-            #         gr.update(visible=False)           # hide NPS modal at start
-            #     )
-
-        
-            #     # try:
-            #     #     print("📄 Processing accession:", acc)
-            #     #     rows = summarize_results(acc)
-            #     #     all_rows.extend(rows)
-            #     #     processed_accessions += 1  # ✅ only count success
-            #     #     if email.strip():
-            #     #         save_to_excel(all_rows, "", "", output_file_path, is_resume=False)
-            #     #     log_lines.append(f"✅ Processed {acc} ({i+1}/{total})")
-            #     print("📄 Processing accession:", acc)
-            #     # --- Before calling summarize_results ---
-            #     samples_left = total - i  # including current one
-            #     estimated_seconds_left = samples_left * 100  # your observed average per sample
-                
-            #     log_lines.append(
-            #         f"Running... usually ~100s per sample"
-            #     )
-            #     log_lines.append(
-            #         f"⏳ Estimated time left: ~{estimated_seconds_left} seconds ({samples_left} sample{'s' if samples_left > 1 else ''} remaining)"
-            #     )
-                
-            #     # Yield update to UI before the heavy pipeline call
-            #     yield (
-            #         make_html_table(all_rows),
-            #         gr.update(visible=True),          # results_group
-            #         gr.update(visible=False),         # download_file
-            #         "",                               # usage_display
-            #         "⏳ Processing...",                # status
-            #         "\n".join(log_lines),             # progress_box
-            #         gr.update(visible=False),         # run_button
-            #         gr.update(visible=True),          # stop_button
-            #         gr.update(visible=True),          # reset_button
-            #         gr.update(visible=False),         # raw_text
-            #         gr.update(visible=False),         # file_upload
-            #         gr.update(value=processed_info, visible=True),  # processed_info
-            #         gr.update(visible=False)          # hide NPS modal
-            #     )
-
-            #     # Run summarize_results in a separate process with stop flag support
-            #     success, rows = run_with_timeout(
-            #         summarize_results,
-            #         args=(acc,),
-            #         timeout=None,              # or set max seconds per sample if you want
-            #         stop_value=global_stop_flag
-            #     )
-                
-            #     # If stop was pressed during this accession
-            #     if not success and global_stop_flag.value:
-            #         log_lines.append(f"🛑 Cancelled {acc} before completion")
-            #         # yield (
-            #         #     make_html_table(all_rows),
-            #         #     gr.update(visible=True),
-            #         #     gr.update(visible=False),
-            #         #     "",
-            #         #     "🛑 Stopped",
-            #         #     "\n".join(log_lines)
-            #         # )
-            #         yield (
-            #             make_html_table(all_rows),
-            #             gr.update(visible=True),                           # results_group
-            #             gr.update(value=output_file_path, visible=bool(output_file_path)),  # download_file
-            #             gr.update(value=usage_text, visible=True),         # usage_display
-            #             "🛑 Stopped",                                       # "✅ Done" or "🛑 Stopped"
-            #             "\n".join(log_lines),
-            #             gr.update(visible=False),                          # run_button
-            #             gr.update(visible=False),                          # stop_button
-            #             gr.update(visible=True),                           # reset_button
-            #             gr.update(visible=False),                          # raw_text
-            #             gr.update(visible=False),                          # file_upload
-            #             gr.update(value="", visible=False),                # processed_info
-            #             gr.update(visible=True)                            # NPS modal now visible
-            #         )
 
-            #         break  # stop processing entirely
-                
-            #     # If it finished normally
-            #     if success and rows:
-            #         all_rows.extend(rows)
-            #         processed_accessions += 1
-            #         if email.strip():
-            #             save_to_excel(all_rows, "", "", output_file_path, is_resume=False)
-            #         log_lines.append(f"✅ Processed {acc} ({i+1}/{total})")
-            #     else:
-            #         # If it failed due to timeout or other error
-            #         if not global_stop_flag.value:
-            #             log_lines.append(f"⚠️ Skipped {acc} due to timeout or error")
+                except Exception as e:
+                    log_lines.append(f"❌ Failed to process {acc}: {e}. Report on the box above so that we won't count this bad one for you (email required).")
+                    yield (
+                        make_html_table(all_rows),         # output_table
+                        gr.update(visible=True),          # results_group
+                        gr.update(visible=False),          # download_file
+                        "",                                # usage_display
+                        "⏳ Processing...",                 # status
+                        "\n".join(log_lines),              # progress_box
+                        gr.update(visible=True),           # run_button
+                        gr.update(visible=True),           # stop_button
+                        gr.update(visible=True),           # reset_button
+                        gr.update(visible=False),          # hide raw_text
+                        gr.update(visible=False),          # hide file_upload
+                        gr.update(value=processed_info, visible=True),  # processed_info
+                        gr.update(visible=False)           # hide NPS modal at start
+                    )
                 
-            #     # Always yield updated logs after each attempt
-            #     # yield (
-            #     #     make_html_table(all_rows),
-            #     #     gr.update(visible=True),
-            #     #     gr.update(visible=False),
-            #     #     "",
-            #     #     "⏳ Processing...",
-            #     #     "\n".join(log_lines)
-            #     # )
-            #     yield (
-            #         make_html_table(all_rows),         # output_table
-            #         gr.update(visible=True),          # results_group
-            #         gr.update(visible=False),          # download_file
-            #         "",                                # usage_display
-            #         "⏳ Processing...",                 # status
-            #         "\n".join(log_lines),              # progress_box
-            #         gr.update(visible=True),           # run_button
-            #         gr.update(visible=True),           # stop_button
-            #         gr.update(visible=True),           # reset_button
-            #         gr.update(visible=False),          # hide raw_text
-            #         gr.update(visible=False),          # hide file_upload
-            #         gr.update(value=processed_info, visible=True),  # processed_info
-            #         gr.update(visible=False)           # hide NPS modal at start
-            #     )
-            
-
-                # except Exception as e:
-                #     log_lines.append(f"❌ Failed to process {acc}: {e}")
-        
-                # yield (
-                #     make_html_table(all_rows),
-                #     gr.update(visible=True),
-                #     gr.update(visible=False),
-                #     "",
-                #     "⏳ Processing...",
-                #     "\n".join(log_lines)
-                # )
-        
             # Step 3: Final usage update
             usage_text = ""
             if email.strip() and not email_tracked: