Update pipeline.py

pipeline.py

@@ -287,13 +287,18 @@ async def process_link_chunk_allOutput(link, iso, acc, saveLinkFolder, out_links
             asyncio.to_thread(data_preprocess.extract_table, link, saveLinkFolder),
             timeout=10
         )
+        print("this is table link: ", str(table_links))
+
     except Exception:
         tables_link = []
 
     # --- merge ---
     try:
         print("just merge text and tables")
-        final_input_link = text_link + ", ".join(tables_link)
+        try:
+          final_input_link = text_link + ", ".join(tables_link)
+        except:  
+          final_input_link = str(text_link) + str(tables_link)
     except Exception:
         print("no succeed here in preprocess docu")
         final_input_link = ""
@@ -306,9 +311,9 @@ async def process_link_chunk_allOutput(link, iso, acc, saveLinkFolder, out_links
     if len(final_input_link) > 1000000:
         final_input_link = data_preprocess.normalize_for_overlap(final_input_link)
         if len(final_input_link) > 1000000:
-            final_input_link = final_input_link[:100000]
+            final_input_link = final_input_link[:1000000]
 
-    all_output = data_preprocess.normalize_for_overlap(all_output) + final_input_link
+    all_output += data_preprocess.normalize_for_overlap(all_output) + final_input_link
 
     return context, all_output, chunk
 
@@ -802,8 +807,8 @@ async def pipeline_with_gemini(accessions,stop_flag=None, niche_cases=None, save
 
           # combine results
           for context, new_all_output, new_chunk in results:
-              all_output = new_all_output
-              chunk = new_chunk  
+              all_output += new_all_output
+              chunk += new_chunk  
           # for link in links:
           #     print(link)
           #     # if len(all_output) > 1000*1000: