First version

.gitattributes

@@ -0,0 +1 @@
+static/images/flowchart.png filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -0,0 +1,25 @@
+# Use a small Python base image
+FROM python:3.10
+
+RUN useradd -m -u 1000 user
+USER user
+# Fast, clean installs
+ENV PYTHONDONTWRITEBYTECODE=1 \
+    PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1 \
+    PORT=7860
+
+WORKDIR /app
+
+# Install Python deps
+COPY --chown=user ./requirements.txt requirements.txt
+RUN pip install --no-cache-dir --upgrade -r requirements.txt
+
+# Copy your app
+COPY --chown=user . /app
+
+# Expose the port Hugging Face expects
+EXPOSE 7860
+
+# Run your app (it will read $PORT below)
+CMD ["python", "app.py"]

app.py

@@ -0,0 +1,214 @@
+import io, uuid, time, os
+import flask
+from flask_socketio import SocketIO, emit, join_room, leave_room
+import eventlet
+from core.mtdna_backend import *
+
+# accessions = []
+isvip = True  # or True depending on the user
+
+app = flask.Flask(__name__)
+app.config["DEBUG"] = True
+app.config["SECRET_KEY"] = "dev-key"
+socketio = SocketIO(app, async_mode="eventlet", cors_allowed_origins="*")
+
+# --- job registry for cancel flags ---
+# Use a simple boolean flag in eventlet mode: True => cancel requested
+CANCEL_FLAGS = {}  # {job_id: bool}
+JOBS = {} # {job_id: {"accs": [...], "started": False}}
+
+# Home
+@app.route("/")
+def home():
+    return flask.render_template("Home.html", isvip=isvip)
+
+# Submit route
+@app.route("/submit", methods=["POST"])
+def submit():
+    raw_text = flask.request.form.get("raw_text", "").strip()
+    file_upload = flask.request.files.get("file_upload")
+    user_email = flask.request.form.get("user_email", "").strip()
+
+    if file_upload and getattr(file_upload, "filename", ""):
+        data = file_upload.read()
+        buf = io.BytesIO(data); buf.name = file_upload.filename
+        file_upload = buf
+
+    accs, error = extract_accessions_from_input(file=file_upload, raw_text=raw_text)
+
+    job_id = uuid.uuid4().hex[:8]
+    CANCEL_FLAGS[job_id] = False
+
+    # Obtain user's past usage
+    user_hash = hash_user_id(user_email)
+    user_usage, max_allowed = increment_usage(user_hash, 0) # get how much they have run and the maximun #queries they have
+    remaining_trials = max(max_allowed - user_usage, 0) # remaining trials if everything goes well
+    total_queries = max(0, min(len(accs), max_allowed - user_usage)) # limited the number of queries of users so that won't have to run all.
+    
+    # the list of IDs that will be run within allowance
+    accs = accs[:total_queries]
+
+    # Save var to the global environment
+    JOBS[job_id] = {"accs": accs, 
+                    "user": {
+                        "user_hash": user_hash, 
+                        "user_usage": user_usage, 
+                        "max_allowed": max_allowed,
+                        "total_queries": total_queries,
+                        "remaining_trials": remaining_trials
+                        }, 
+                    "started": False} 
+
+    return flask.redirect(flask.url_for("output", job_id=job_id))
+
+# Output page (must accept job_id!)
+@app.route("/output/<job_id>")
+def output(job_id):
+    started_at = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime())
+    total_queries = JOBS[job_id]['user']['total_queries']
+    return flask.render_template(
+        "Output.html",
+        job_id=job_id,
+        started_at=started_at,
+        total_queries=total_queries,
+        isvip=isvip,
+        ws_url="",  # leave empty for mock/demo mode; set later to real WS URL
+    )
+
+# Functions that communicates between web and server - run via socketio
+def run_job(job_id, accessions):
+    total_queries = JOBS[job_id]['user']['total_queries']
+    user_hash = JOBS[job_id]['user']['user_hash'] # to update allowance in case cancelled
+    room = job_id
+
+
+    def send_log(msg): socketio.emit("log", {"msg": msg}, room=room)
+    def send_row(row): socketio.emit("row", row, room=room)
+
+    
+    try:
+        socketio.emit("status", {"state": "started", "total": len(accessions)}, room=room)
+        start_time = time.perf_counter()
+        send_log(f"Job {job_id} started. {total_queries} accession(s).")
+
+        outs = []
+        for i, acc in enumerate(accessions, 1):
+            if CANCEL_FLAGS.get(job_id):
+                send_log("Cancellation requested. Stopping…")
+                socketio.emit("status", {"state": "cancelled"}, room=room)
+                increment_usage(user_hash, i - 1)
+                return
+
+            t0 = time.perf_counter()
+            out = summarize_results(acc)  # may be dict / [] / None
+            dt = time.perf_counter() - t0
+
+            # ---- normalise 'out' to a dict we can emit safely ----
+            if not out:
+                out = {}
+            elif isinstance(out, list):
+                # If a list slipped through, try to coerce sensibly
+                if out and isinstance(out[0], dict):
+                    out = out[0]
+                elif out and isinstance(out[0], list):
+                    # very defensive: list-of-lists -> map by expected order
+                    keys = [
+                        "Sample ID", "Predicted Country", "Country Explanation",
+                        "Predicted Sample Type", "Sample Type Explanation",
+                        "Sources", "Time cost"
+                    ]
+                    row0 = out[0]
+                    out = {k: (row0[idx] if idx < len(row0) else "") for idx, k in enumerate(keys)}
+                else:
+                    out = {}
+            elif not isinstance(out, dict):
+                out = {}
+
+            # ---- map backend keys (Title Case) to frontend keys (snake_case) ----
+            sample_id                 = out.get("Sample ID") or str(acc)
+            predicted_country         = out.get("Predicted Country", "unknown")
+            country_explanation       = out.get("Country Explanation", "unknown")
+            predicted_sample_type     = out.get("Predicted Sample Type", "unknown")
+            sample_type_explanation   = out.get("Sample Type Explanation", "unknown")
+            sources                   = out.get("Sources", "No Links")
+            time_cost                 = out.get("Time cost") or f"{dt:.2f}s"
+
+            send_row = {
+                "idx": i,
+                "sample_id": sample_id,
+                "predicted_country": predicted_country,
+                "country_explanation": country_explanation,
+                "predicted_sample_type": predicted_sample_type,
+                "sample_type_explanation": sample_type_explanation,
+                "sources": sources,
+                "time_cost": time_cost,
+            }
+
+            socketio.emit("row", send_row, room=room)
+            socketio.sleep(0)  # <- correct spelling; yield so the emit flushes
+            send_log(f"Processed {acc} in {dt:.2f}s")
+
+        total_dt = time.perf_counter() - start_time
+        
+        # Update user allowance
+        increment_usage(user_hash, total_queries)
+        # Calculate remaining_trials to display for user
+        remaining_trials = JOBS[job_id]['user']['remaining_trials']
+
+        socketio.emit("status", {"state": "finished", "elapsed": f"{total_dt:.2f}s"}, room=room)
+        send_log(f"Job finished successfully. Number of trials left is")
+    except Exception as e:
+        send_log(f"ERROR: {e}")
+        socketio.emit("status", {"state": "error", "message": str(e)}, room=room)
+    finally:
+        CANCEL_FLAGS.pop(job_id, None)
+        JOBS.pop(job_id, None)  # <— tidy queued job
+
+# ---- Socket.IO events ----
+@socketio.on("connect")
+def on_connect():
+    emit("connected", {"ok": True})
+
+@socketio.on("join")
+def on_join(data):
+    job_id = data.get("job_id")
+    if job_id:
+        join_room(job_id)
+        emit("joined", {"room": job_id})
+
+        # Start the job once the client is in the room
+        job = JOBS.get(job_id)
+        if job and not job["started"]:
+            job["started"] = True
+            total = len(job["accs"])
+            # Send an initial queued/total status so the UI can set progress denominator
+            socketio.emit("status", {"state": "queued", "total": total}, room=job_id)
+            socketio.start_background_task(run_job, job_id, job["accs"])
+
+@socketio.on("leave")
+def on_leave(data):
+    job_id = data.get("job_id")
+    if job_id:
+        leave_room(job_id)
+
+@socketio.on("cancel")
+def on_cancel(data):
+    job_id = data.get("job_id")
+    if job_id in CANCEL_FLAGS:
+        CANCEL_FLAGS[job_id] = True  # flip the flag
+        emit("status", {"state": "cancelling"}, room=job_id)
+
+@app.route("/about")
+def about():
+    return flask.render_template("About.html", isvip=isvip)
+
+@app.route("/pricing")
+def pricing():
+    return flask.render_template("Pricing.html", isvip=isvip)
+
+@app.route("/contact")
+def contact():
+    return flask.render_template("Contact.html", isvip=isvip)
+
+port = int(os.environ.get("PORT", 7860))  # HF Spaces injects PORT
+socketio.run(app, host="0.0.0.0", port=port)

core/DefaultPackages/__init__.py

@@ -0,0 +1,4 @@
+__all__ = [
+        'openFile',
+        'saveFile',
+        ]

core/DefaultPackages/openFile.py

@@ -0,0 +1,12 @@
+def openFile(file):
+  with open(file) as f:
+    openFile = f.read()
+  return openFile
+
+def openJsonFile(file):
+  import json
+  # Opening JSON file
+  with open(file, 'r') as openfile:
+    # Reading from json file
+    json_object = json.load(openfile)
+  return json_object

core/DefaultPackages/saveFile.py

@@ -0,0 +1,11 @@
+import json
+def saveFile(name,content):
+  Name = name
+  fi = open(Name, "a")
+      # Add new change in to saved file
+  with open(Name, "w") as external_file:
+    add_text = content
+    print(add_text, file=external_file)
+    external_file.close()
+def saveJsonFile(name,content):
+  saveFile(name,json.dumps(content))    

core/NER/PDF/pdf.py

@@ -0,0 +1,193 @@
+#!pip install pdfreader
+import pdfreader
+from pdfreader import PDFDocument, SimplePDFViewer
+#!pip install bs4
+from bs4 import BeautifulSoup
+import requests
+from core.NER import cleanText
+#!pip install tabula-py
+import tabula
+import fitz  # PyMuPDF
+import os
+
+class PDF():
+  def __init__(self, pdf, saveFolder, doi=None):
+    self.pdf = pdf
+    self.doi = doi
+    self.saveFolder = saveFolder
+
+  def openPDFFile(self):
+    if "https" in self.pdf:
+      name = self.pdf.split("/")[-1]
+      name = self.downloadPDF(self.saveFolder)
+      if name != "no pdfLink to download":
+        fileToOpen = os.path.join(self.saveFolder, name)
+      else:
+        fileToOpen = self.pdf
+    else:
+      fileToOpen = self.pdf
+    return open(fileToOpen, "rb")
+
+  def downloadPDF(self, saveFolder):
+    pdfLink = ''
+    if ".pdf" not in self.pdf and "https" not in self.pdf:
+      r = requests.get(self.pdf)
+      soup = BeautifulSoup(r.content, 'html.parser')
+      links = soup.find_all("a")
+      for link in links:
+        if ".pdf" in link.get("href", ""):
+          if self.doi in link.get("href"):
+            pdfLink = link.get("href")
+            break
+    else:
+      pdfLink = self.pdf
+
+    if pdfLink != '':
+      response = requests.get(pdfLink)
+      name = pdfLink.split("/")[-1]
+      print("inside download PDF and name and link are: ", pdfLink, name)  
+      print("saveFolder is: ", saveFolder)  
+      with open(os.path.join(saveFolder, name), 'wb') as pdf:
+        print("len of response content: ", len(response.content))  
+        pdf.write(response.content)
+      print("pdf downloaded")
+      return name
+    else:
+      return "no pdfLink to download"
+
+  def extractText(self):
+    try:  
+        fileToOpen = self.openPDFFile().name
+        try:
+          doc = fitz.open(fileToOpen)
+          text = ""
+          for page in doc:
+            text += page.get_text("text") + "\n\n"
+          doc.close()
+    
+          if len(text.strip()) < 100:
+            print("Fallback to PDFReader due to weak text extraction.")
+            text = self.extractTextWithPDFReader()
+          return text
+        except Exception as e:
+          print("Failed with PyMuPDF, fallback to PDFReader:", e)
+          return self.extractTextWithPDFReader()
+    except:
+        return ""
+  def extract_text_excluding_tables(self):
+    fileToOpen = self.openPDFFile().name
+    text = ""
+    try:
+        doc = fitz.open(fileToOpen)
+        for page in doc:
+            blocks = page.get_text("dict")["blocks"]
+            
+            for block in blocks:
+                if block["type"] == 0:  # text block
+                    lines = block.get("lines", [])
+                    
+                    if not lines:
+                        continue
+                    avg_words_per_line = sum(len(l["spans"]) for l in lines) / len(lines)
+                    if avg_words_per_line > 1:  # Heuristic: paragraph-like blocks
+                        for line in lines:
+                            text += " ".join(span["text"] for span in line["spans"]) + "\n"
+        doc.close()
+        if len(text.strip()) < 100:
+          print("Fallback to PDFReader due to weak text extraction.")
+          text = self.extractTextWithPDFReader()
+        return text
+    except Exception as e:
+      print("Failed with PyMuPDF, fallback to PDFReader:", e)
+      return self.extractTextWithPDFReader()
+
+  def extractTextWithPDFReader(self):
+    jsonPage = {}
+    try:
+        pdf = self.openPDFFile()
+        print("open pdf file")  
+        print(pdf)  
+        doc = PDFDocument(pdf)
+        viewer = SimplePDFViewer(pdf)
+        all_pages = [p for p in doc.pages()]
+        cl = cleanText.cleanGenText()
+        pdfText = ""
+        for page in range(1, len(all_pages)):
+          viewer.navigate(page)
+          viewer.render()
+          if str(page) not in jsonPage:
+            jsonPage[str(page)] = {}
+          text = "".join(viewer.canvas.strings)
+          clean, filteredWord = cl.textPreprocessing(text)
+          jsonPage[str(page)]["normalText"] = [text]
+          jsonPage[str(page)]["cleanText"] = [' '.join(filteredWord)]
+          jsonPage[str(page)]["image"] = [viewer.canvas.images]
+          jsonPage[str(page)]["form"] = [viewer.canvas.forms]
+          jsonPage[str(page)]["content"] = [viewer.canvas.text_content]
+          jsonPage[str(page)]["inline_image"] = [viewer.canvas.inline_images]
+        pdf.close()
+    except:
+        jsonPage = {}        
+    return self.mergeTextinJson(jsonPage)
+
+  def extractTable(self,pages="all",saveFile=None,outputFormat=None):
+    '''pages (str, int, iterable of int, optional) –
+      An optional values specifying pages to extract from. It allows str,`int`, iterable of :int. Default: 1
+      Examples: '1-2,3', 'all', [1,2]'''
+    df = []
+    if "https" in self.pdf:
+      name = self.pdf.split("/")[-1]
+      name = self.downloadPDF(self.saveFolder)
+      if name != "no pdfLink to download":
+        fileToOpen = self.saveFolder + "/" + name
+      else: fileToOpen = self.pdf
+    else: fileToOpen = self.pdf
+    try:
+      df = tabula.read_pdf(fileToOpen, pages=pages)
+    # saveFile: "/content/drive/MyDrive/CollectData/NER/PDF/tableS1.csv"
+    # outputFormat: "csv"
+    #tabula.convert_into(self.pdf, saveFile, output_format=outputFormat, pages=pages)
+    except:# ValueError:
+      df = []
+      print("No tables found in PDF file")
+    return df
+
+  def mergeTextinJson(self, jsonPDF):
+    try:  
+        cl = cleanText.cleanGenText()
+        pdfText = ""
+        if jsonPDF:  
+            for page in jsonPDF:
+              if len(jsonPDF[page]["normalText"]) > 0:
+                for i in range(len(jsonPDF[page]["normalText"])):
+                  text = jsonPDF[page]["normalText"][i]
+                  if len(text) > 0:
+                    text = cl.removeTabWhiteSpaceNewLine(text)
+                    text = cl.removeExtraSpaceBetweenWords(text)
+                  jsonPDF[page]["normalText"][i] = text
+                  if i - 1 > 0:
+                    if jsonPDF[page]["normalText"][i - 1][-1] != ".":
+                      pdfText += ". "
+                  pdfText += jsonPDF[page]["normalText"][i]
+                if len(jsonPDF[page]["normalText"][i]) > 0:
+                  if jsonPDF[page]["normalText"][i][-1] != ".":
+                    pdfText += "."
+                pdfText += "\n\n"
+        return pdfText
+    except:
+        return ""
+
+  def getReference(self):
+    pass
+
+  def getSupMaterial(self):
+    pass
+
+  def removeHeaders(self):
+    pass
+
+  def removeFooters(self):
+    pass
+
+  def removeReference(self):
+    pass

core/NER/WordDoc/wordDoc.py

@@ -0,0 +1,178 @@
+#! pip install spire.doc
+#! pip install Spire.XLS
+import pandas as pd
+from spire.doc import *
+from spire.doc.common import *
+from spire.xls import *
+from spire.xls.common import *
+from core.NER import cleanText
+import requests 
+class wordDoc(): # using python-docx
+  def __init__(self, wordDoc,saveFolder):
+    self.wordDoc = wordDoc
+    self.saveFolder = saveFolder
+  def openFile(self):
+    document = Document()
+    return document.LoadFromFile(self.wordDoc)
+  def extractTextByPage(self):
+    # reference: https://medium.com/@alice.yang_10652/extract-text-from-word-documents-with-python-a-comprehensive-guide-95a67e23c35c#:~:text=containing%20specific%20content.-,Spire.,each%20paragraph%20using%20the%20Paragraph.
+    json = {}
+    #doc = self.openFile()
+    # Create an object of the FixedLayoutDocument class and pass the Document object to the class constructor as a parameter
+    try:
+      doc = Document()
+      doc.LoadFromFile(self.wordDoc)
+    except:
+      response = requests.get(self.wordDoc)
+      name = self.wordDoc.split("/")[-1]
+      with open(self.saveFolder+"/" + name, "wb") as temp_file:  # Create a temporary file to store the downloaded data
+        temp_file.write(response.content)  
+      doc = Document()
+      doc.LoadFromFile(self.saveFolder+"/" + name)
+    text = doc.GetText()
+    return text
+  def extractTableAsText(self):
+    getDoc = ''
+    try:
+      # reference:
+      # https://www.e-iceblue.com/Tutorials/Python/Spire.Doc-for-Python/Program-Guide/Table/Python-Extract-Tables-from-Word-Documents.html?gad_source=1&gclid=Cj0KCQiA6Ou5BhCrARIsAPoTxrCj3XSsQsDziwqE8BmVlOs12KneOlvtKnn5YsDruxK_2T_UUhjw6NYaAtJhEALw_wcB
+      doc = Document()
+      doc.LoadFromFile(self.wordDoc)
+      getDoc = "have document"
+    except:
+      response = requests.get(self.wordDoc)
+      name = self.wordDoc.split("/")[-1]
+      with open(self.saveFolder+"/" + name, "wb") as temp_file:  # Create a temporary file to store the downloaded data
+        temp_file.write(response.content)  
+      doc = Document()
+      doc.LoadFromFile(self.saveFolder+"/" + name)  
+      getDoc = "have document"
+    json = {}
+    if len(getDoc) > 0:
+      # Loop through the sections
+      for s in range(doc.Sections.Count):
+        # Get a section
+          section = doc.Sections.get_Item(s)
+          # Get the tables in the section
+          json["Section" + str(s)] = {}
+          tables = section.Tables
+          # Loop through the tables
+          for i in range(0, tables.Count):
+              # Get a table
+              table = tables.get_Item(i)
+              # Initialize a string to store the table data
+              tableData = ''
+              # Loop through the rows of the table
+              for j in range(0, table.Rows.Count):
+                  # Loop through the cells of the row
+                  for k in range(0, table.Rows.get_Item(j).Cells.Count):
+                      # Get a cell
+                      cell = table.Rows.get_Item(j).Cells.get_Item(k)
+                      # Get the text in the cell
+                      cellText = ''
+                      for para in range(cell.Paragraphs.Count):
+                          paragraphText = cell.Paragraphs.get_Item(para).Text
+                          cellText += (paragraphText + ' ')
+                      # Add the text to the string
+                      tableData += cellText
+                      if k < table.Rows.get_Item(j).Cells.Count - 1:
+                          tableData += '\t'
+                  # Add a new line
+                  tableData += '\n'
+              json["Section" + str(s)]["Table"+str(i)] = tableData
+    return json
+  def extractTableAsList(self):
+    tables = []
+    try:
+      doc = Document()
+      doc.LoadFromFile(self.wordDoc)
+    except:
+      response = requests.get(self.wordDoc)
+      name = self.wordDoc.split("/")[-1]
+      with open(os.path.join(self.saveFolder, name), "wb") as f:
+        f.write(response.content)
+      doc = Document()
+      doc.LoadFromFile(os.path.join(self.saveFolder, name))
+
+    for s in range(doc.Sections.Count):
+      section = doc.Sections.get_Item(s)
+      for i in range(section.Tables.Count):
+        table = section.Tables.get_Item(i)
+        table_data = []
+        for row in range(table.Rows.Count):
+          row_data = []
+          for cell in range(table.Rows.get_Item(row).Cells.Count):
+            cell_obj = table.Rows.get_Item(row).Cells.get_Item(cell)
+            cell_text = ""
+            for p in range(cell_obj.Paragraphs.Count):
+              cell_text += cell_obj.Paragraphs.get_Item(p).Text.strip() + " "
+            row_data.append(cell_text.strip())
+          table_data.append(row_data)
+        tables.append(table_data)
+    return tables  
+  def extractTableAsExcel(self):
+    getDoc = ''
+    try:
+      # reference:
+      # https://www.e-iceblue.com/Tutorials/Python/Spire.Doc-for-Python/Program-Guide/Table/Python-Extract-Tables-from-Word-Documents.html?gad_source=1&gclid=Cj0KCQiA6Ou5BhCrARIsAPoTxrCj3XSsQsDziwqE8BmVlOs12KneOlvtKnn5YsDruxK_2T_UUhjw6NYaAtJhEALw_wcB
+      doc = Document()
+      doc.LoadFromFile(self.wordDoc)
+      getDoc = "have document"
+    except:
+      response = requests.get(self.wordDoc)
+      name = self.wordDoc.split("/")[-1]
+      with open(self.saveFolder+"/" + name, "wb") as temp_file:  # Create a temporary file to store the downloaded data
+        temp_file.write(response.content)  
+      doc = Document()
+      doc.LoadFromFile(self.saveFolder+"/" + name)  
+      getDoc = "have document"
+    if len(getDoc) > 0:
+      try:
+        # Create an instance of Workbook
+        wb = Workbook()
+        wb.Worksheets.Clear()
+
+        # Loop through sections in the document
+        for i in range(doc.Sections.Count):
+            # Get a section
+            section = doc.Sections.get_Item(i)
+            # Loop through tables in the section
+            for j in range(section.Tables.Count):
+                # Get a table
+                table = section.Tables.get_Item(j)
+                # Create a worksheet
+                ws = wb.Worksheets.Add(f'Table_{i+1}_{j+1}')
+                # Write the table to the worksheet
+                for row in range(table.Rows.Count):
+                    # Get a row
+                    tableRow = table.Rows.get_Item(row)
+                    # Loop through cells in the row
+                    for cell in range(tableRow.Cells.Count):
+                        # Get a cell
+                        tableCell = tableRow.Cells.get_Item(cell)
+                        # Get the text in the cell
+                        cellText = ''
+                        for paragraph in range(tableCell.Paragraphs.Count):
+                            paragraph = tableCell.Paragraphs.get_Item(paragraph)
+                            cellText = cellText + (paragraph.Text + ' ')
+                        # Write the cell text to the worksheet
+                        ws.SetCellValue(row + 1, cell + 1, cellText)
+
+        # Save the workbook
+        name = self.wordDoc.split("/")[-1]
+        if self.saveFolder == None:
+          wb.SaveToFile('/content/drive/MyDrive/CollectData/NER/excel/TestExamples/output/'+name+".xlsx", FileFormat.Version2016)
+          nameFile = '/content/drive/MyDrive/CollectData/NER/excel/TestExamples/output/'+name+".xlsx"
+        else:
+          wb.SaveToFile(self.saveFolder+'/'+name+".xlsx", FileFormat.Version2016)
+          nameFile = self.saveFolder+'/'+name + ".xlsx"
+        doc.Close()
+        wb.Dispose()
+        return nameFile
+      except: return "No table found on word doc"  
+    else:
+      return "No table found on word doc"     
+  def getReference(self):
+    pass
+  def getSupMaterial(self):
+    pass

core/NER/cleanText.py

@@ -0,0 +1,115 @@
+# reference:
+# https://ayselaydin.medium.com/1-text-preprocessing-techniques-for-nlp-37544483c007
+import re, json
+import nltk
+#nltk.download('stopwords')
+#nltk.download()
+from core.DefaultPackages import openFile, saveFile
+from nltk.corpus import stopwords
+from nltk.corpus.reader.api import wordpunct_tokenize
+from nltk.tokenize import word_tokenize
+#from wordsegment import load, segment
+from wordsegment import load, segment
+class cleanGenText():
+  def __init__(self):
+    #self.text = text
+    load()
+    pass
+  def removePunct(self,text,KeepPeriod=False):
+    punctuation = r'[^\w\s]'
+    if KeepPeriod==True:
+      punctuation = r'[^\w\s\.]' 
+    return re.sub(punctuation, '', text)
+  def removeURL(self,text):
+    url_pattern = re.compile(r'https?://\S+|www\.\S+')
+    return url_pattern.sub(r'', text)
+  def removeHTMLTag(self,text):
+    html_tags_pattern = r'<.*?>'
+    return re.sub(html_tags_pattern, '', text)
+  def removeTabWhiteSpaceNewLine(self,text):
+    # remove \n or \t and unnecessary white space
+    cleanText = text.replace("\n\n","")
+    cleanText = text.replace("\n","")
+    cleanText = cleanText.replace("\t","")
+    cleanText = cleanText.strip()
+    return cleanText
+  def removeExtraSpaceBetweenWords(self,text):
+    return re.sub(r'\s+', ' ',text).strip()  
+  def removeStopWords(self,text):
+    #extraUnwantedWords = ["resource","groups","https","table","online","figure","frequency","aslo","fig","shows","respectively"]
+    filteredWord = []
+    stopWords = set(list(set(stopwords.words('english'))))# + extraUnwantedWords)
+    textWords = word_tokenize(text)
+    for word in textWords:
+      if word.lower() not in stopWords:
+        filteredWord.append(word) # and w.isalpha()==True]
+    return filteredWord
+  def removeLowercaseBetweenUppercase(self,segment):
+    # segment such as "Myanmar (formerly Burma)"
+    # but not change anything for "Viet Nam"
+    # for special cases:
+        # the capital letter:
+        # When there is a lowercase word between:
+        # e.g: "Myanmar (formerly Burma)" can be "Myanmar", "Burma" instead of "myanmar formerly burma"        
+        # When there is no lowercase word or uppercase words in a row:
+        # e.g: "Viet Nam" can be "Viet Nam" or "viet nam", instead of "Viet", "Nam"
+    outputUp = []
+    segment = self.removeTabWhiteSpaceNewLine(segment)
+    segments = segment.split(" ")
+    for w in range(len(segments)):
+      word = segments[w]
+      cleanWord = self.removePunct(word)
+      cleanWord = self.removeTabWhiteSpaceNewLine(cleanWord) 
+      prevWord = ""
+      if w > 0:
+        prevWord = segments[w-1]
+        cleanPreWord = self.removePunct(prevWord)
+        cleanPreWord = self.removeTabWhiteSpaceNewLine(cleanPreWord)
+      if cleanWord[0].isupper() == True: # check isupper of first letter of capital word
+        if len(prevWord)>0 and prevWord[0].isupper() == True:
+          outputUp[-1] += " " + cleanWord 
+        else:
+          outputUp.append(cleanWord)
+    return outputUp    
+  def textPreprocessing(self, text, keepPeriod=False):
+    # lowercase
+    #lowerText = self.text.lower()
+    # remove punctuation & special characacters
+    cleanText = self.removePunct(text, KeepPeriod=keepPeriod)
+    # removal of URLs in text
+    cleanText = self.removeURL(cleanText)
+    # removal of HTML Tags
+    cleanText = self.removeHTMLTag(cleanText)
+    # remove \n or \t and unnecessary white space
+    cleanText = self.removeTabWhiteSpaceNewLine(cleanText)
+    # stop-words removal
+    filteredWord = self.removeStopWords(cleanText)
+    # a sentence or the capital word behind a period "."
+    return cleanText, filteredWord
+  #generateNewChar = textPreprocessing("/content/drive/MyDrive/CollectData/NER/CountriesNameNCBI.json")
+  #saveFile.saveFile("/content/drive/MyDrive/CollectData/NER/NewCharCountriesNameNCBI.json", json.dumps(generateNewChar))
+  def splitStickWords(self,word):
+    #output = []
+    split_words = segment(word)
+    '''for w in split_words:
+      pos = word.lower().find(w)
+      if word[pos].isupper() == True:
+        output.append(w[0].upper() + w[1:])   
+      else:
+        output.append(w)
+      if pos >=0:
+        if pos+len(w)<len(word):
+          if word[pos+len(w)] == ".":   
+            output[-1] = output[-1] + "."  '''
+    return " ".join(split_words)
+  def removeDOI(self, word, doiLink=None):
+    # if they have the word DOI in that: ex: 1368598DOI after general clean
+    if "DOI" in word:
+      word = word.replace(word,"")
+    # if they have the link DOI in that: ex: 10.1007s004390161742yORIGINAL, but we still split the word
+    if doiLink != None:
+      w = self.splitStickWords(word)
+      cleanDOI = self.removePunct(doiLink)
+      if cleanDOI in w:
+        word = w.replace(cleanDOI,"")
+    return word

core/NER/html/extractHTML.py

@@ -0,0 +1,226 @@
+# reference: https://www.crummy.com/software/BeautifulSoup/bs4/doc/#for-html-documents
+from bs4 import BeautifulSoup
+import requests
+from core.DefaultPackages import openFile, saveFile
+from core.NER import cleanText
+import pandas as pd
+class HTML():
+  def __init__(self, htmlFile, htmlLink):
+    self.htmlLink = htmlLink
+    self.htmlFile = htmlFile
+  # def openHTMLFile(self):
+  #   headers = {
+  #       "User-Agent": (
+  #           "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+  #           "AppleWebKit/537.36 (KHTML, like Gecko) "
+  #           "Chrome/114.0.0.0 Safari/537.36"
+  #       ),
+  #       "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+  #       "Referer": self.htmlLink,
+  #       "Connection": "keep-alive"
+  #   }
+
+  #   session = requests.Session()
+  #   session.headers.update(headers)
+
+  #   if self.htmlLink != "None":
+  #       try:
+  #           r = session.get(self.htmlLink, allow_redirects=True, timeout=15)
+  #           if r.status_code != 200:
+  #               print(f"❌ HTML GET failed: {r.status_code} — {self.htmlLink}")
+  #               return BeautifulSoup("", 'html.parser')
+  #           soup = BeautifulSoup(r.content, 'html.parser')
+  #       except Exception as e:
+  #           print(f"❌ Exception fetching HTML: {e}")
+  #           return BeautifulSoup("", 'html.parser')
+  #   else:
+  #       with open(self.htmlFile) as fp:
+  #           soup = BeautifulSoup(fp, 'html.parser')
+  #   return soup
+  from lxml.etree import ParserError, XMLSyntaxError
+  
+  def openHTMLFile(self):
+      not_need_domain = ['https://broadinstitute.github.io/picard/',
+              'https://software.broadinstitute.org/gatk/best-practices/',
+              'https://www.ncbi.nlm.nih.gov/genbank/',
+              'https://www.mitomap.org/']
+      headers = {
+          "User-Agent": (
+              "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+              "AppleWebKit/537.36 (KHTML, like Gecko) "
+              "Chrome/114.0.0.0 Safari/537.36"
+          ),
+          "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+          "Referer": self.htmlLink,
+          "Connection": "keep-alive"
+      }
+
+      session = requests.Session()
+      session.headers.update(headers)
+      if self.htmlLink in not_need_domain:
+        return BeautifulSoup("", 'html.parser')
+      try:
+          if self.htmlLink and self.htmlLink != "None":
+              r = session.get(self.htmlLink, allow_redirects=True, timeout=15)
+              if r.status_code != 200 or not r.text.strip():
+                  print(f"❌ HTML GET failed ({r.status_code}) or empty page: {self.htmlLink}")
+                  return BeautifulSoup("", 'html.parser')
+              soup = BeautifulSoup(r.content, 'html.parser')
+          else:
+              with open(self.htmlFile, encoding='utf-8') as fp:
+                  soup = BeautifulSoup(fp, 'html.parser')
+      except (ParserError, XMLSyntaxError, OSError) as e:
+          print(f"🚫 HTML parse error for {self.htmlLink}: {type(e).__name__}")
+          return BeautifulSoup("", 'html.parser')
+      except Exception as e:
+          print(f"❌ General exception for {self.htmlLink}: {e}")
+          return BeautifulSoup("", 'html.parser')
+
+      return soup
+
+  def getText(self):
+    soup = self.openHTMLFile()
+    s = soup.find_all("html")
+    text = ""
+    if s:
+      for t in range(len(s)):
+        text = s[t].get_text()
+    cl = cleanText.cleanGenText()
+    text = cl.removeExtraSpaceBetweenWords(text)
+    return text
+  def getListSection(self, scienceDirect=None):
+    try:  
+        json = {}
+        text = ""
+        textJson, textHTML = "",""
+        if scienceDirect == None:
+          soup = self.openHTMLFile()
+          # get list of section
+          json = {}
+          for h2Pos in range(len(soup.find_all('h2'))):
+            if soup.find_all('h2')[h2Pos].text not in json:
+              json[soup.find_all('h2')[h2Pos].text] = []
+            if h2Pos + 1 < len(soup.find_all('h2')):
+              content = soup.find_all('h2')[h2Pos].find_next("p")
+              nexth2Content = soup.find_all('h2')[h2Pos+1].find_next("p")
+              while content.text != nexth2Content.text:
+                json[soup.find_all('h2')[h2Pos].text].append(content.text)
+                content = content.find_next("p")
+            else:
+              content = soup.find_all('h2')[h2Pos].find_all_next("p",string=True)
+              json[soup.find_all('h2')[h2Pos].text] = list(i.text for i in content)
+          # format
+          '''json = {'Abstract':[], 'Introduction':[], 'Methods'[],
+            'Results':[], 'Discussion':[], 'References':[],
+            'Acknowledgements':[], 'Author information':[], 'Ethics declarations':[],
+            'Additional information':[], 'Electronic supplementary material':[],
+            'Rights and permissions':[], 'About this article':[], 'Search':[], 'Navigation':[]}'''
+        if scienceDirect!= None or len(json)==0:
+          # Replace with your actual Elsevier API key
+          api_key = os.environ["SCIENCE_DIRECT_API"]  
+          # ScienceDirect article DOI or PI (Example DOI)
+          doi =  self.htmlLink.split("https://doi.org/")[-1]  #"10.1016/j.ajhg.2011.01.009"
+          # Base URL for the Elsevier API
+          base_url = "https://api.elsevier.com/content/article/doi/"
+          # Set headers with API key
+          headers = {
+              "Accept": "application/json",
+              "X-ELS-APIKey": api_key
+          }
+          # Make the API request
+          response = requests.get(base_url + doi, headers=headers)
+    # Check if the request was successful
+          if response.status_code == 200:
+            data = response.json()
+            supp_data = data["full-text-retrieval-response"]#["coredata"]["link"]
+            if "originalText" in list(supp_data.keys()):
+              if type(supp_data["originalText"])==str:
+                json["originalText"] = [supp_data["originalText"]]
+              if type(supp_data["originalText"])==dict:
+                json["originalText"] = [supp_data["originalText"][key] for key in supp_data["originalText"]]
+            else:
+              if type(supp_data)==dict:
+                for key in supp_data:
+                  json[key] = [supp_data[key]]
+    
+        textJson = self.mergeTextInJson(json)
+        textHTML = self.getText()
+        if len(textHTML) > len(textJson):
+          text = textHTML
+        else: text = textJson
+        return text #json
+    except:
+        print("failed all")
+        return ""  
+  def getReference(self):
+    # get reference to collect more next data
+    ref = []
+    json = self.getListSection()
+    for key in json["References"]:
+      ct = cleanText.cleanGenText(key)
+      cleanText, filteredWord = ct.cleanText()
+      if cleanText not in ref:
+        ref.append(cleanText)
+    return ref
+  def getSupMaterial(self):
+    # check if there is material or not
+    json = {}
+    soup = self.openHTMLFile()
+    for h2Pos in range(len(soup.find_all('h2'))):
+      if "supplementary" in soup.find_all('h2')[h2Pos].text.lower() or "material" in soup.find_all('h2')[h2Pos].text.lower() or "additional" in soup.find_all('h2')[h2Pos].text.lower() or "support" in soup.find_all('h2')[h2Pos].text.lower():
+        #print(soup.find_all('h2')[h2Pos].find_next("a").get("href"))
+        link, output = [],[]
+        if soup.find_all('h2')[h2Pos].text not in json:
+          json[soup.find_all('h2')[h2Pos].text] = []
+        for l in soup.find_all('h2')[h2Pos].find_all_next("a",href=True):
+            link.append(l["href"])
+        if h2Pos + 1 < len(soup.find_all('h2')):
+          nexth2Link = soup.find_all('h2')[h2Pos+1].find_next("a",href=True)["href"]
+          if nexth2Link in link:
+            link = link[:link.index(nexth2Link)]
+        # only take links having "https" in that
+        for i in link:
+          if "https" in i:  output.append(i)
+        json[soup.find_all('h2')[h2Pos].text].extend(output)
+    return json
+  def extractTable(self):
+    soup = self.openHTMLFile()
+    df = []
+    if len(soup)>0:
+      try:
+        df = pd.read_html(str(soup))
+      except ValueError:
+        df = []
+        print("No tables found in HTML file")
+    return df
+  def mergeTextInJson(self,jsonHTML):
+    cl = cleanText.cleanGenText()
+    #cl = cleanGenText()
+    htmlText = ""
+    for sec in jsonHTML:
+      # section is "\n\n"
+      if len(jsonHTML[sec]) > 0:
+        for i in range(len(jsonHTML[sec])):
+          # same section is just a dot.
+          text = jsonHTML[sec][i]
+          if len(text)>0:
+            #text = cl.removeTabWhiteSpaceNewLine(text)
+            #text = cl.removeExtraSpaceBetweenWords(text)
+            text, filteredWord = cl.textPreprocessing(text, keepPeriod=True)
+          jsonHTML[sec][i] = text
+          if i-1 >= 0:
+            if len(jsonHTML[sec][i-1])>0:
+              if jsonHTML[sec][i-1][-1] != ".":
+                htmlText += ". "
+          htmlText += jsonHTML[sec][i]
+        if len(jsonHTML[sec][i]) > 0:
+          if jsonHTML[sec][i][-1]!=".":
+            htmlText += "."
+        htmlText += "\n\n"
+    return htmlText
+  def removeHeaders(self):
+    pass
+  def removeFooters(self):
+    pass
+  def removeReferences(self):
+    pass

core/NER/word2Vec/heuristic.py

@@ -0,0 +1,52 @@
+import logging
+from datetime import datetime
+
+class HeuristicManager:
+    def __init__(self, model, log_file="heuristic_log.txt", min_similarity_threshold=0.5, min_new_data_len=50):
+        self.model = model
+        self.min_similarity_threshold = min_similarity_threshold
+        self.min_new_data_len = min_new_data_len
+        self.log_file = log_file
+        logging.basicConfig(filename=self.log_file, level=logging.INFO)
+
+    def log(self, message):
+        timestamp = datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        logging.info(f"[{timestamp}] {message}")
+        print(f"[{timestamp}] {message}")
+
+    def check_similarity(self, test_terms):
+        triggers = []
+        for term in test_terms:
+            try:
+                sim = self.model.wv.most_similar(term)[0][1]
+                if sim < self.min_similarity_threshold:
+                    triggers.append(f"Low similarity for '{term}': {sim}")
+            except KeyError:
+                triggers.append(f"'{term}' not in vocabulary")
+        return triggers
+
+    def check_metadata(self, metadata):
+        triggers = []
+        if any(keyword in str(metadata).lower() for keyword in ["haplogroup b", "eastasia", "asian"]):
+            triggers.append("Detected new haplogroup or regional bias: 'Asian' or 'B'")
+        return triggers
+
+    def check_new_data_volume(self, new_data):
+        if len(new_data) < self.min_new_data_len:
+            return ["Not enough new data to justify retraining"]
+        return []
+
+    def should_retrain(self, test_terms, new_data, metadata):
+        triggers = []
+        triggers += self.check_similarity(test_terms)
+        triggers += self.check_metadata(metadata)
+        triggers += self.check_new_data_volume(new_data)
+
+        if triggers:
+            self.log("Retraining triggered due to:")
+            for trigger in triggers:
+                self.log(f" - {trigger}")
+            return True
+        else:
+            self.log("No retraining needed.")
+            return False

core/NER/word2Vec/testModel/test_model.txt

@@ -0,0 +1,25 @@
+24 100
+dna -0.0005385255 0.0002430238 0.005111818 0.009016951 -0.009293036 -0.007109866 0.0064572324 0.008987154 -0.0050192317 -0.0037659889 0.0073785 -0.0015431087 -0.0045221853 0.006557529 -0.004854595 -0.0018278129 0.002881375 0.0010002495 -0.00829578 -0.009462763 0.007312361 0.0050688535 0.0067577288 0.0007685764 0.006347226 -0.003397316 -0.0009421973 0.0057741464 -0.007532499 -0.0039303782 -0.0075064874 -0.0009439946 0.009533595 -0.0073319245 -0.002333888 -0.0019326513 0.0080786925 -0.005930193 3.549824e-05 -0.00475331 -0.0095964745 0.005000012 -0.008770563 -0.0043735923 -2.9246534e-05 -0.00030931013 -0.007669701 0.009599569 0.004982613 0.009233704 -0.008148657 0.004488859 -0.0041414667 0.00081141765 0.008487031 -0.00446156 0.0045125154 -0.006793622 -0.0035560841 0.009394251 -0.0015774865 0.00032431752 -0.004129968 -0.0076763057 -0.0015165819 0.0024841889 -0.00088440755 0.0055526863 -0.0027446826 0.002259023 0.0054701897 0.008356409 -0.0014508999 -0.009201209 0.004375452 0.00058271736 0.0074576377 -0.00080706284 -0.0026372937 -0.008752899 -0.00087625836 0.00282087 0.005398569 0.0070530027 -0.0057170955 0.0018605916 0.006099475 -0.0048024287 -0.003104349 0.0067992285 0.0016360026 0.00019302641 0.00348545 0.00021818833 0.009630539 0.0050670514 -0.008908632 -0.007042295 0.0009007676 0.0063867364
+from -0.00861988 0.0036778022 0.005193427 0.005744547 0.0074751326 -0.0061739217 0.0011082628 0.0060625207 -0.0028567386 -0.006184132 -0.00041290926 -0.008384168 -0.0055893976 0.007104685 0.003362318 0.007228353 0.0068033817 0.007533677 -0.003792071 -0.000581891 0.0023577819 -0.0045196284 0.008395244 -0.009858517 0.006761404 0.0029261683 -0.004930935 0.0043925527 -0.0017370671 0.006713542 0.009974645 -0.0043735756 -0.0006050642 -0.005716478 0.003858548 0.002799571 0.00690247 0.00610934 0.009526547 0.009269763 0.007910428 -0.007008808 -0.00916451 -0.00033672128 -0.0030898354 0.007890073 0.005923819 -0.001552973 0.001516021 0.0017856265 0.007822941 -0.009514211 -0.00020886083 0.0034666678 -0.00094713847 0.008384139 0.009009283 0.0065234327 -0.0007208324 0.007705209 -0.00853289 0.0032079336 -0.004625999 -0.0050743804 0.0035901158 0.005388813 0.007766254 -0.005744939 0.0074327383 0.006626378 -0.003704473 -0.008735958 0.005445474 0.0065230317 -0.000784768 -0.006700798 -0.007075852 -0.002488528 0.0051543443 -0.0036620772 -0.00938257 0.003815971 0.004890136 -0.0064404616 0.0012033634 -0.0020763231 2.994902e-05 -0.0098790005 0.002700701 -0.004756241 0.0011076172 -0.0015674155 0.0022046466 -0.00787344 -0.0027070795 0.002668326 0.0053478787 -0.002396734 -0.009512201 0.0045024394
+mtdna 8.645293e-05 0.003076037 -0.006815487 -0.0013743688 0.0076927417 0.0073529496 -0.0036715195 0.0026677884 -0.008309281 0.00619759 -0.00463892 -0.0031715294 0.009313415 0.00088058383 0.0074962615 -0.00608139 0.005167896 0.009930803 -0.008471472 -0.0051321597 -0.007057574 -0.0048644566 -0.003772668 -0.008518714 0.0079532955 -0.0048361127 0.008438283 0.005270068 -0.0065578814 0.0039592343 0.005482614 -0.007444929 -0.0074228924 -0.002492343 -0.008628872 -0.0015748737 -0.00038757667 0.0032959366 0.0014325404 -0.00088083016 -0.005591098 0.0017297626 -0.00089552783 0.0068030986 0.0039881677 0.004533183 0.0014284542 -0.0027126821 -0.0043595196 -0.0010315293 0.0014437438 -0.0026617546 -0.0070882514 -0.007825746 -0.009136036 -0.005931676 -0.001850123 -0.004323682 -0.0064626597 -0.0037265678 0.004296681 -0.0037233941 0.008404572 0.001539496 -0.007246572 0.009443451 0.007636867 0.0055208146 -0.0068550883 0.0058190743 0.004034045 0.005188155 0.0042629624 0.0019477821 -0.003167882 0.008342064 0.009619138 0.0038047181 -0.0028461283 5.6938893e-07 0.0012001555 -0.0084682545 -0.008234347 -0.00023238244 0.0012304098 -0.005750644 -0.0047139754 -0.0073490315 0.008316314 0.00010242269 -0.004513882 0.005704978 0.009199796 -0.004097329 0.007985275 0.005386452 0.0058861696 0.0005043713 0.008208188 -0.0070221694
+in -0.008226077 0.009303831 -0.00018710589 -0.0019704443 0.0046143015 -0.004104392 0.0027394402 0.006979235 0.0060486975 -0.0075411424 0.00939576 0.00465202 0.004012172 -0.006245291 0.008499353 -0.002164537 0.008836197 -0.005347778 -0.008136817 0.006804632 0.0016640095 -0.0022142953 0.009522269 0.009494823 -0.0097868545 0.0025105644 0.0061560757 0.0038842657 0.0020310257 0.00043876152 0.00068163266 -0.0038464246 -0.007141551 -0.0020813115 0.003930752 0.008838634 0.009274302 -0.0059668766 -0.009419525 0.009759848 0.0034291998 0.005158939 0.006265811 -0.0027623416 0.007310359 0.0027998323 0.0028576967 -0.0023982434 -0.003139742 -0.0023701421 0.0042809984 4.8589092e-05 -0.009614385 -0.00968607 -0.006160773 -0.00011437661 0.0019819876 0.009428 0.0056011924 -0.004298171 0.00026028603 0.004974084 0.007744428 -0.001135339 0.004278759 -0.0057750097 -0.0008068469 0.00811882 -0.002369315 -0.009674972 0.0058119837 -0.0039038642 -0.001220125 0.010017389 -0.002241946 -0.0047185957 -0.0053141676 0.0069846674 -0.005741993 0.002120917 -0.0052751247 0.00613608 0.0043662013 0.0026298608 -0.0015129133 -0.002735619 0.008999614 0.0052172863 -0.0021470466 -0.009465257 -0.007413552 -0.0010587372 -0.00078251073 -0.0025414668 0.009710779 -0.00044944565 0.005915 -0.007467981 -0.0024928953 -0.005583053
+european -0.007147033 0.0012623417 -0.007189088 -0.0022513974 0.0037773554 0.005857864 0.0012027922 0.0021598793 -0.004109796 0.007198152 -0.006319537 0.0046250015 -0.008186181 0.0020334523 -0.0049318667 -0.0042960607 -0.0030848773 0.0056965156 0.0057683894 -0.004991361 0.00076802005 -0.008515792 0.0078122346 0.009295911 -0.002746969 0.0008081935 0.0007694419 0.00550255 -0.008630911 0.0006062931 0.0068933573 0.0021813295 0.0010798875 -0.009366349 0.008471645 -0.006258249 -0.0029761735 0.0035168754 -0.00078163494 0.0014152499 0.0017921324 -0.006839617 -0.009737293 0.009092817 0.0062128166 -0.00694695 0.0033956417 0.00017217748 0.004755041 -0.0071203653 0.004067516 0.004303939 0.009927 -0.0045391554 -0.0014395243 -0.0073114103 -0.009704934 -0.009090646 -0.0010375449 -0.0065315044 0.0048550633 -0.006148244 0.0026037877 0.000752482 -0.0034296552 -0.00092229253 0.010017935 0.009206015 -0.004494388 0.009070265 -0.0055859834 0.0059493524 -0.0030818144 0.0034673577 0.003029479 0.0069394265 -0.0023470228 0.008820008 0.0075530927 -0.009551933 -0.008064042 -0.007652859 0.0029148757 -0.0027951996 -0.00694831 -0.008136711 0.008356287 0.0019903474 -0.00933717 -0.004817203 0.0031394493 -0.0046995636 0.005327329 -0.0042287502 0.0027155946 -0.008033582 0.0062630265 0.0047997306 0.00079031993 0.0029888113
+common -0.008722234 0.0021272295 -0.0008539916 -0.009321866 -0.0094246445 -0.001412531 0.0044288053 0.00372704 -0.006505282 -0.006894708 -0.0049991854 -0.0023061878 -0.007229156 -0.009607243 -0.0027377736 -0.008360431 -0.0060269493 -0.005675304 -0.00234906 -0.0017278373 -0.008954683 -0.000731004 0.008155364 0.007693106 -0.007208155 -0.003644954 0.0031189725 -0.009568674 0.0014795078 0.0065395026 0.0057490384 -0.008770905 -0.0045228535 -0.008156553 4.5400484e-05 0.00927559 0.005980464 0.0050585535 0.0050439127 -0.0032448657 0.009562716 -0.0073605715 -0.0072781076 -0.002255642 -0.00077679846 -0.0032283778 -0.00060498127 0.007476424 -0.00070291053 -0.0016193221 0.002749461 -0.008367007 0.0078366995 0.008528508 -0.009591924 0.0024459555 0.009891981 -0.007673955 -0.006969234 -0.0077365288 0.008389148 -0.00067644875 0.009162579 -0.008137346 0.0037369097 0.0026538277 0.0007320811 0.002340243 -0.007473436 -0.009367513 0.0023810826 0.0061679846 0.007993824 0.005740968 -0.00078188477 0.008307063 -0.009312772 0.0033975116 0.00027130058 0.003872196 0.007375048 -0.0067289495 0.005584901 -0.0095183 -0.0008194822 -0.008691651 -0.0050952802 0.009296191 -0.0018460032 0.0029113942 0.009088126 0.008946764 -0.008196811 -0.0030016953 0.009896215 0.005113277 -0.0015862831 -0.008699891 0.0029696936 -0.0066840183
+sequence 0.008134779 -0.0044588344 -0.0010699655 0.001010431 -0.00018677961 0.0011458534 0.0061133304 -1.2402037e-05 -0.0032534893 -0.0015101052 0.0058955555 0.0015073137 -0.0007181427 0.009341042 -0.004917502 -0.0008413052 0.009177319 0.0067567485 0.0015022643 -0.0088886535 0.0011522508 -0.0022903979 0.009365224 0.0012041465 0.0014943897 0.0024040388 -0.0018358674 -0.004996856 0.00023002276 -0.0020175653 0.0066060103 0.008935089 -0.0006746635 0.0029776676 -0.0061099143 0.0017025766 -0.006924371 -0.008690522 -0.005899618 -0.008961226 0.0072769034 -0.005776607 0.00827455 -0.007233702 0.003422895 0.009676102 -0.0077943387 -0.009949275 -0.0043248134 -0.0026828882 -0.0002740396 -0.008833413 -0.008620106 0.0027985822 -0.008205106 -0.009067738 -0.0023404285 -0.00863584 -0.007056119 -0.008398832 -0.0003011976 -0.0045611723 0.006630901 0.0015288803 -0.0033471577 0.006116343 -0.0060124504 -0.004648673 -0.0072044823 -0.0043340866 -0.0018032556 0.00649206 -0.0027680297 0.004921421 0.006912646 -0.007459126 0.004573438 0.006129695 -0.002956148 0.0066218316 0.006121442 -0.0064460207 -0.0067676785 0.002543585 -0.0016248615 -0.006062931 0.009498339 -0.005135456 -0.006549685 -0.000118091535 -0.002699267 0.00044816377 -0.0035289875 -0.00041692218 -0.00070437486 0.00083035015 0.0081978375 -0.005737508 -0.0016556873 0.005569238
+bru18 0.008155276 -0.0044185193 0.008987652 0.008259665 -0.0044238693 0.00031090993 0.004277394 -0.0039252234 -0.0055654007 -0.006509729 -0.0006656875 -0.00030213682 0.004489389 -0.0024855223 -0.00015437756 0.0024471143 0.0048732683 -2.8606542e-05 -0.0063628056 -0.009279111 1.8654398e-05 0.006667726 0.0014650559 -0.0089674555 -0.007945727 0.006548857 -0.0037690091 0.006254232 -0.0067004655 0.008482541 -0.0065189763 0.0032740948 -0.001067833 -0.0067885593 -0.0032949874 -0.0011434925 -0.005471747 -0.001204045 -0.0075744605 0.0026601462 0.009080238 -0.0023750134 -0.0009867329 0.0035252234 0.008680149 -0.0059299506 -0.006889695 -0.002942458 0.00913801 0.0008666254 -0.008663911 -0.001442217 0.009477263 -0.0075691855 -0.0053729587 0.009308613 -0.008970956 0.0038234547 0.00065334333 0.0066515543 0.008311967 -0.002862157 -0.003982641 0.008891435 0.0020839446 0.0062542376 -0.009450494 0.0095988605 -0.0013514485 -0.006062315 0.0029950105 -0.0004512243 0.0047055846 -0.0022705523 -0.004145877 0.0022992992 0.008370594 -0.004990823 0.0026696166 -0.00798221 -0.0067810714 -0.000469271 -0.008768882 0.0027844147 0.0015907697 -0.0023179457 0.005011737 0.009743466 0.008472866 -0.001870301 0.0020416898 -0.0039901678 -0.008234559 0.0062697986 -0.0019247098 -0.00066059735 -0.0017619281 -0.004536765 0.004069 -0.0042896206
+bru50 -0.009579504 0.008948466 0.0041579367 0.00923892 0.006649052 0.0029269105 0.009801864 -0.0044190143 -0.0068119396 0.004226486 0.0037328962 -0.005664456 0.009715384 -0.0035591167 0.009558758 0.00083636935 -0.006334789 -0.0019748765 -0.007390546 -0.002990235 0.0010405012 0.009480547 0.009361016 -0.0065955063 0.0034724285 0.0022746115 -0.0024764987 -0.009228658 0.0010185506 -0.008164371 0.0063289437 -0.0058100903 0.005530614 0.009826734 -0.00015984276 0.0045368825 -0.0018012718 0.0073676347 0.0039300686 -0.0090082595 -0.0023973046 0.0036249864 -0.00010732573 -0.0011888575 -0.0010430571 -0.0016724848 0.00059902505 0.0041630277 -0.004250072 -0.0038341933 -5.2427928e-05 0.00026678806 -0.00017553278 -0.0047934647 0.0043008197 -0.002173452 0.0020970574 0.00065915886 0.005959963 -0.0068526124 -0.00680708 -0.004473089 0.009448878 -0.001590459 -0.009438289 -0.000534792 -0.0044530216 0.0060103727 -0.009585406 0.002857136 -0.009246552 0.001258808 0.0059965253 0.0074065947 -0.007623657 -0.0060443347 -0.006831209 -0.007910946 -0.009496376 -0.0021281417 -0.0008362788 -0.007265241 0.0067816544 0.0011141741 0.0058228294 0.0014675015 0.00078702695 -0.007366497 -0.0021715113 0.0043177926 -0.005089294 0.001137756 0.0028883398 -0.0015285894 0.009943532 0.008348668 0.0024183327 0.007110643 0.005890512 -0.005592114
+vietnam -0.005153963 -0.0066644135 -0.007776157 0.0083126435 -0.0019782323 -0.006856599 -0.004155673 0.0051580225 -0.0028790692 -0.0037560624 0.0016262402 -0.00278304 -0.001570952 0.0010760438 -0.002967586 0.008515032 0.003917556 -0.009953211 0.0062494674 -0.0067655 0.00076895714 0.0043992978 -0.005096968 -0.0021128112 0.00809259 -0.0042428537 -0.0076304777 0.009258844 -0.0021577128 -0.004717085 0.008580298 0.004269408 0.004324098 0.009280228 -0.008452614 0.0052631963 0.0020472223 0.004193831 0.0016919046 0.004460046 0.0044873925 0.0060984488 -0.0032084621 -0.0045590503 -0.0004232687 0.002529075 -0.0032731881 0.006051339 0.0041546253 0.00776509 0.002568826 0.008108382 -0.0013972289 0.008070817 0.003707151 -0.008045609 -0.00393531 -0.0024772724 0.004889826 -0.00087688275 -0.00282919 0.007839672 0.009338199 -0.0016121961 -0.0051723607 -0.0046861414 -0.0048465827 -0.0095901145 0.0013706182 -0.0042283125 0.002539541 0.0056244545 -0.00406352 -0.009583576 0.0015531465 -0.006689678 0.0025049727 -0.0037749638 0.007073151 0.00063951715 0.0035553342 -0.0027433916 -0.001711565 0.007655947 0.0014000075 -0.005851 -0.007834303 0.0012315387 0.006458937 0.0055561876 -0.00897213 0.008598417 0.0040550055 0.007476387 0.00975736 -0.007282407 -0.009030263 0.0058277464 0.009392481 0.0034955258
+sample 0.007100903 -0.0015709094 0.007947078 -0.00948947 -0.00802812 -0.006650821 -0.004002562 0.00500194 -0.0038224515 -0.008330948 0.00841617 -0.0037529538 0.008619977 -0.004892141 0.003931126 0.004920354 0.0023956115 -0.0028135795 0.0028564015 -0.008257614 -0.0027645228 -0.0026008752 0.007249391 -0.0034709626 -0.0066022277 0.0043369113 -0.0004823991 -0.0035912786 0.006893536 0.003869671 -0.0038965137 0.0007677057 0.009145668 0.0077625574 0.0063656354 0.004670941 0.0023901698 -0.0018358309 -0.006370667 -0.00030689163 -0.0015674513 -0.00057719386 -0.0062623145 0.0074473424 -0.0066001806 -0.007243944 -0.0027626618 -0.0015170419 -0.007635178 0.0006969715 -0.005330137 -0.0012829994 -0.007370956 0.0019601034 0.003276234 -1.4737604e-05 -0.005451358 -0.001723771 0.00709824 0.003738 -0.008888436 -0.0034084066 0.0023648455 0.0021412992 -0.009477984 0.004583573 -0.008656226 -0.007383396 0.0034825006 -0.0034719554 0.0035707187 0.008896884 -0.003571185 0.009332037 0.0017215977 0.009857596 0.005704204 -0.009146731 -0.0033407472 0.0065290304 0.0055978918 0.008714949 0.0069304765 0.008049887 -0.009821734 0.004303451 -0.0050309277 0.0035138857 0.0060621244 0.0043927776 0.007520648 0.0014953684 -0.0012639741 0.0057787485 -0.0056348047 4.0551466e-05 0.009468461 -0.005486985 0.0038199269 -0.008121091
+collected 0.0097750295 0.008170629 0.0012814446 0.0051154387 0.0014172737 -0.006454876 -0.0014259414 0.0064561926 -0.004619688 -0.0039992593 0.004923175 0.0027045405 -0.0018415204 -0.0028716852 0.006021755 -0.005721393 -0.003250512 -0.0064803455 -0.0042360183 -0.008592084 -0.004467861 -0.008505252 0.0013975133 -0.008609542 -0.009919709 -0.008202052 -0.0067797694 0.006683116 0.0037784956 0.0003495915 -0.002959815 -0.007438984 0.0005348175 0.0005005026 0.00019596443 0.0008583165 0.00078985846 -5.4285138e-05 -0.008013045 -0.005872034 -0.00837931 -0.0013207265 0.0018039295 0.0074345516 -0.001966708 -0.0023440684 0.009481904 7.425008e-05 -0.0023982543 0.008607863 0.0026964454 -0.0053582233 0.0065950346 0.0045082304 -0.0070585674 -0.00031050213 0.00083163293 0.005739447 -0.0017207591 -0.0028131874 0.0017429565 0.00085032795 0.0012085037 -0.002637083 -0.0060016937 0.007339091 0.0075857476 0.00830421 -0.008602928 0.0026385786 -0.0035621128 0.0096288975 0.0029010975 0.004643974 0.0023910597 0.006626162 -0.005746352 0.007899223 -0.0024186398 -0.0045691207 -0.0020768652 0.009735589 -0.0068560173 -0.0021970137 0.006994984 -4.366915e-05 -0.0062879827 -0.006398747 0.008941079 0.0064397687 0.004773856 -0.003261329 -0.009269935 0.0038002136 0.0071752095 -0.0056398017 -0.007860231 -0.0029721109 -0.0049388385 -0.0023143636
+europe -0.0019466967 -0.005264445 0.009446078 -0.009301849 0.00450806 0.005410841 -0.0014122794 0.009008321 0.009883694 -0.0054709506 -0.0060238987 -0.006749262 -0.007891144 -0.0030501 -0.00559189 -0.008350158 0.000785714 0.002999436 0.0064088805 -0.0026336086 -0.0044599404 0.0012484614 0.00038998463 0.008114584 0.00018636887 0.0072303875 -0.008259172 0.008436813 -0.0018950498 0.008705898 -0.007616939 0.0017924334 0.0010528992 4.4615095e-05 -0.005109563 -0.009249746 -0.0072665187 -0.007951877 0.0019136231 0.00048003704 -0.0018163731 0.007123826 -0.0024782037 -0.0013449806 -0.008898934 -0.0099250255 0.008953352 -0.0057566464 -0.006378906 0.0052002883 0.0066733453 -0.0068328637 0.000956345 -0.0060142023 0.0016413335 -0.004295812 -0.0034417375 0.0021831726 0.008657248 0.0067267795 -0.00967649 -0.0056275628 0.007884859 0.0019889344 -0.0042598336 0.0006024022 0.009526292 -0.0011015745 -0.009430234 0.0016114928 0.0062343916 0.00628738 0.0040935944 -0.0056507527 -0.000374705 -4.9610684e-05 0.004579015 -0.0080420235 -0.008019654 0.0002663556 -0.008607854 0.005816331 -0.00042231655 0.00997148 -0.0053460747 -0.00048954826 0.0077552027 -0.004073562 -0.0050113807 0.0015921831 0.0026467363 -0.0025611357 0.006453244 -0.0076659652 0.003398472 0.00049256504 0.008736541 0.0059848153 0.006820848 0.007819741
+ancient -0.00949331 0.009558393 -0.0077741044 -0.0026378995 -0.0048897555 -0.0049655624 -0.008022211 -0.007766241 -0.0045622233 -0.0012816157 -0.0051147 0.0061208857 -0.009519694 -0.005296118 0.009434444 0.0069931676 0.0076746074 0.0042455657 0.0005105317 -0.0060022003 0.006030395 0.002638317 0.007692142 0.0063923756 0.0079497155 0.008663229 -0.009898174 -0.006753931 0.0013303582 0.0064388 0.0073839277 0.0055065546 0.007657052 -0.0051452103 0.006578382 -0.004109781 -0.009049926 0.009156881 0.0013312489 -0.0027684697 -0.0024686211 -0.004237798 0.004802247 0.00442113 -0.0026455545 -0.0073452652 -0.0035828727 -0.00034474322 0.006112652 -0.0028318586 -0.00011603545 0.0008713841 -0.007088451 0.0020616641 -0.0014378024 0.0028043352 0.0048393123 -0.0013679614 -0.0027919079 0.0077378284 0.005049118 0.006718327 0.0045309924 0.00867961 0.0074680797 -0.0010581953 0.008750674 0.0046186065 0.0054406407 -0.0013790869 -0.0020325198 -0.0044157715 -0.008505952 0.0030342783 0.008892043 0.0089222565 -0.0019243953 0.0060931933 0.0037896668 -0.0043041655 0.002026212 -0.005454141 0.008199508 0.005422219 0.003183278 0.0041012214 0.008660769 0.007268954 -0.0008326238 -0.0070764753 0.008396081 0.0072427383 0.0017482204 -0.0013339228 -0.0058783586 -0.004530154 0.008643081 -0.003131084 -0.006341318 0.009878559
+neanderthal 0.007692736 0.009126856 0.001134214 -0.008323363 0.008438394 -0.0036978398 0.005743373 0.0044079996 0.0096743805 -0.009301011 0.009201668 -0.009297726 -0.0068989955 -0.009099583 -0.0055382987 0.0073707746 0.009167804 -0.0033190295 0.0037136457 -0.0036417823 0.007886165 0.0058672884 4.5112392e-06 -0.0036315187 -0.0072244583 0.0047761244 0.0014634884 -0.002615084 0.007832942 -0.004045295 -0.00913638 -0.0022702827 0.00011177889 -0.006659164 -0.0054871286 -0.008484606 0.00924395 0.0074312175 -0.00030530593 0.0073675984 0.0079630045 -0.0007988404 0.0066030715 0.0037836921 0.0050928146 0.0072574555 -0.004751798 -0.0021930316 0.00087973 0.0042327694 0.0033078827 0.0050869007 0.004582786 -0.008444151 -0.0031969673 -0.007233252 0.009679768 0.0049946425 0.0001599608 0.0041068383 -0.0076482734 -0.0062929546 0.003092239 0.006544919 0.0039503933 0.006035828 -0.0019895614 -0.0033235473 0.00020525315 -0.0031931365 -0.005507259 -0.0077802544 0.0065467777 -0.0010795805 -0.0018928167 -0.007799526 0.009349405 0.00087477046 0.0017788016 0.0024914553 -0.0073950374 0.0016234348 0.0029714536 -0.008580277 0.0049522887 0.0024255016 0.0074964412 0.0050449395 -0.0030210917 -0.0071717766 0.007105708 0.0019140064 0.005210298 0.0063858717 0.0019259832 -0.0061174775 -5.528207e-06 0.008260976 -0.0060965912 0.009431074
+modern -0.0071792696 0.0042354544 0.00216289 0.007438057 -0.0048900596 -0.0045788498 -0.0060949842 0.0033097882 -0.004507435 0.008506253 -0.0042799306 -0.009108578 -0.0047961376 0.0064152437 -0.006351414 -0.0052630682 -0.007296127 0.006024725 0.003365447 0.0028487756 -0.0031356772 0.00602019 -0.0061529716 -0.001984372 -0.0059886468 -0.0009987217 -0.0020279228 0.008489572 9.179515e-05 -0.0085772425 -0.0054273363 -0.0068765874 0.0026914866 0.00946441 -0.0058075436 0.008274624 0.008538083 -0.007054826 -0.008883825 0.009470304 0.008378029 -0.0046964334 -0.0067229234 0.007853816 0.003754884 0.008087255 -0.0075793806 -0.009526273 0.0015759452 -0.009809055 -0.004886255 -0.003462314 0.009610498 0.008620381 -0.002831389 0.005837147 0.008235405 -0.002257783 0.009542199 0.0071611865 0.0020309114 -0.0038430467 -0.005072538 -0.00304804 0.007877576 -0.0061799455 -0.0029184332 0.009190523 0.003460949 0.0060627563 -0.008025261 -0.00075433304 0.0055211782 -0.0046972577 0.0074892025 0.009333807 -0.00041072394 -0.0020574103 -0.00060545607 -0.0057792794 -0.0083910655 -0.0014910942 -0.0025447267 0.0043934747 -0.006866489 0.00542165 -0.006739068 -0.0078106844 0.008480591 0.008917766 -0.0034737175 0.0034897032 -0.005797486 -0.008738294 -0.0055089584 0.0067478465 0.0064329007 0.009427363 0.007059985 0.0067415633
+human 0.0013073076 -0.009817197 0.0046000797 -0.00054215814 0.0063516907 0.0017917434 -0.0031376705 0.00779152 0.0015605913 4.5087592e-05 -0.004629277 -0.008477088 -0.0077653346 0.00868444 -0.0089293 0.009021215 -0.009282701 -0.00026340262 -0.0019013402 -0.008945062 0.008634705 0.006775237 0.0030073978 0.00484689 0.000119797296 0.009438227 0.007017406 -0.009846283 -0.0044378787 -0.0012810889 0.0030511408 -0.0043373024 0.0014413317 -0.007862512 0.002772104 0.0047001 0.004937028 -0.0031820575 -0.008430869 -0.009233454 -0.00072350266 -0.007335406 -0.0068239835 0.006137866 0.0071648457 0.0021028868 -0.00790615 -0.0057202103 0.008053211 0.0039317366 -0.0052275606 -0.007412702 0.00076265965 0.0034572822 0.002076003 0.0031028383 -0.0056280685 -0.0099016195 -0.0070258062 0.00023322599 0.0046109683 0.004535595 0.0018992841 0.0051839855 -0.000116945404 0.004136494 -0.009110944 0.0077172276 0.0061438708 0.0051303217 0.0072363587 0.0084579345 0.00074768433 -0.0017087719 0.0005303956 -0.009314834 0.008429295 -0.0063797934 0.008425091 -0.0042409054 0.0006248087 -0.009168093 -0.009569658 -0.007833339 -0.0077458574 0.00037962993 -0.0072201644 -0.004963075 -0.0052754995 -0.004289475 0.0070301695 0.004834569 0.008708495 0.0070971223 -0.0056847483 0.007253502 -0.009290819 -0.0025857396 -0.007757146 0.0042008474
+genome 0.0018013249 0.0070483726 0.002941503 -0.006984167 0.0077269375 -0.005990631 0.008982948 0.0029859466 -0.0040263417 -0.0046959417 -0.004423949 -0.006166649 0.009397486 -0.0026410713 0.00779025 -0.009682492 0.0021134273 -0.001217051 0.007545118 -0.009060286 0.007431912 -0.005112224 -0.006022511 -0.0056468663 -0.0033655176 -0.0034046597 -0.0031906026 -0.007475777 0.0007148267 -0.0005725245 -0.0016790004 0.0037438255 -0.00763313 -0.0032234066 0.00514847 0.00855509 -0.009791086 0.0071872775 0.0052953 -0.003874173 0.008570203 -0.009222292 0.0072385296 0.0053781155 0.0012898272 -0.0051951176 -0.004179599 -0.003369767 0.0015944163 0.001581598 0.007396833 0.0099602975 0.008836587 -0.004008733 0.009636086 -0.00063042255 0.0048575792 0.0025363516 -0.0006256454 0.0036644523 -0.005330011 -0.0057551167 -0.007577021 0.0019176035 0.006513916 0.00090115983 0.0012633507 0.0031810037 0.008123854 -0.007687061 0.0022752027 -0.007455608 0.003715618 0.009514587 0.0075186947 0.006441567 0.008026117 0.006552105 0.0068467325 0.00869257 -0.0049556913 0.009209661 0.0050575286 -0.0021248695 0.008474546 0.005080482 0.009641399 0.0028190457 0.009884555 0.001195692 0.009130684 0.0035973836 0.006580412 -0.00361116 0.0068057566 0.007250423 -0.002115621 -0.0018615718 0.003625693 -0.0070385
+shows 0.009741375 -0.009785563 -0.006502033 0.0027767855 0.0064354893 -0.005370729 0.0027519849 0.009131747 -0.006819064 -0.0061066505 -0.0049928115 -0.00368126 0.0018522884 0.009683641 0.00644354 0.00039165124 0.0024744181 0.00844649 0.009138178 0.005629969 0.005943013 -0.007629522 -0.0038295696 -0.005683565 0.0061836103 -0.00225932 -0.008786562 0.0076284255 0.008406309 -0.0033179314 0.009119112 -0.00073907804 -0.0036286868 -0.0003802314 0.00019241076 -0.0035078088 0.0028134247 0.005731432 0.006873956 -0.008905951 -0.0021951643 -0.0054816343 0.0075234827 0.0065075015 -0.0043688817 0.002324414 -0.0059516523 0.00023538349 0.00945961 -0.0026105444 -0.0051873005 -0.0074033006 -0.0029152564 -0.0008664178 0.0035291065 0.009743326 -0.0033921245 0.001903681 0.009692432 0.0015337794 0.0009810732 0.009802843 0.00930645 0.007710903 -0.006179333 0.009991138 0.005857104 0.009073708 -0.002001237 0.0033512171 0.0068392376 -0.0038913293 0.006648019 0.0025668114 0.009319553 -0.0030298685 -0.0031094935 0.0062168743 -0.00908894 -0.0072543155 -0.006503641 -0.00074380165 -0.002362113 0.0068256087 0.009239293 -0.00091146474 0.0014132133 0.002020571 -0.0020174456 -0.008035576 0.007445874 -0.004299319 0.004580612 0.009090945 0.0030486963 0.00313993 0.0040727276 -0.0027017219 0.0038345656 0.00033530922
+variation 0.005626712 0.005497371 0.0018291199 0.0057494068 -0.008968078 0.0065593575 0.009225992 -0.0042071473 0.0016075504 -0.0052338815 0.0010582185 0.0027701687 0.008160736 0.00054401276 0.0025570584 0.001297735 0.008402523 -0.0057077026 -0.00626183 -0.0036275184 -0.0023005498 0.005041063 -0.008120357 -0.0028335357 -0.008197427 0.00514971 -0.0025680638 -0.009067107 0.0040717293 0.009017323 -0.0030376601 -0.0058385395 0.0030198884 -0.00043584823 -0.009979436 0.008417704 -0.0073388875 -0.004930407 -0.002657081 -0.0054523144 0.00171651 0.009712814 0.0045722723 0.008088603 -0.00047045827 0.0006449234 -0.002668352 -0.008779561 0.0034313034 0.0020933736 -0.009421854 -0.004968437 -0.009734099 -0.0057197916 0.0040645422 0.008642861 0.00411165 0.0023884643 0.008144778 -0.0011192096 -0.0013977134 -0.008746823 -0.00012579202 -0.0025675725 0.00038607715 0.007279662 -0.0070414604 -0.0039464748 -0.0066646053 -0.0035441148 -0.0033158315 0.002137121 0.0033281683 -0.004957187 -0.0045462907 0.0011386942 0.0054534827 0.0053736498 -0.0029685367 -0.0042665256 -0.005616647 -0.00054498314 0.001946373 0.0015253461 0.0073525296 -0.0027333724 -6.592393e-05 -0.0055276332 -0.0011700654 -0.0077119637 -0.0009593296 0.0013096749 -0.008594744 0.0087485835 -0.009207866 -0.009624677 -0.008511624 0.0073132683 0.0054655685 0.009249462
+haplogroup 0.0025659278 0.00085168 -0.0025371916 0.00934742 0.0028080416 0.0041162586 -0.0011815964 0.00096541416 0.0066110776 -0.00074895076 0.0033208325 -0.00070219487 0.0052740807 0.003645613 0.0026175152 -0.0053456044 -0.004693721 0.004352339 -0.0059164464 -0.00020070269 -0.0006396672 0.0034715144 -0.008427317 0.0088428045 -0.0014485243 -0.005307692 0.0040584584 -0.001898596 -0.007778139 -0.0044734394 -0.0003679351 -0.0089815045 0.0005416724 0.002407686 -0.003227299 0.0025667753 0.0024930644 0.009990179 0.0014140693 0.0020159276 0.0027784512 -0.0020868885 -0.008718105 0.008073382 -0.0019698895 -0.009723993 -0.006550278 -0.0039781313 0.003948964 0.0050270366 0.0061098747 -0.006815141 0.00066107995 -0.0028290635 -0.0052407067 0.006984182 0.0039222264 -0.003121762 -0.008263934 -0.0051569464 -0.00065567193 0.0078113875 0.006122021 -0.008424067 -0.0096058855 0.0071855173 -0.0022900787 -0.0036282074 0.005704672 -0.0058300486 0.005136189 -0.00020829153 -0.0068513798 -0.00030139415 0.006364283 0.009325248 0.0022419153 0.0050703404 -0.0050120936 -0.0008110871 -0.005373588 0.0011743606 -0.0017981603 -0.0036161384 -0.0070382343 0.009639485 0.003012655 -0.0022897385 -0.0041911877 0.0076894285 -0.0064663296 0.0031200873 0.0008309826 0.008321212 0.0068888706 -0.0028947534 0.002593874 -0.0016730811 -0.009431767 -0.0026270088
+h 0.0013225824 0.0065497826 0.009982806 0.009062454 -0.0079781795 0.0065080435 -0.0057147983 -0.0009299061 0.00047654507 0.0065626903 0.0044563343 0.0045750956 0.0095022535 0.00038496728 -0.0060190535 -0.006347197 0.0064362343 -0.005219293 -0.002869563 0.004042792 -0.002286449 -0.006022882 -0.0023193487 0.0012384101 0.0021826315 0.0061027543 -0.005193723 0.003081824 0.0072158594 0.0022087328 0.0054155486 -0.004879429 0.0061283903 -0.007640156 0.0034881763 -0.009306421 -0.0025874602 -0.00905658 -0.0016061858 -0.005364485 -0.0039271545 0.0011356737 0.002771372 -0.0014860439 -0.008151553 -0.0059441784 0.00080055697 -0.0039708167 -0.009422841 -0.0007733177 0.0066586556 0.005949332 -0.0099333245 0.0030846666 -0.006018299 -0.009179041 0.00015740465 -0.0003979007 -0.006993792 -0.0063003623 -0.0024212876 0.0071041975 -0.0074873487 0.0077126683 -0.000499351 0.001135528 0.009489626 0.0047690077 -0.0035878688 0.00373115 0.0035563034 0.0063642766 7.750339e-05 -0.0044055916 0.001321394 -0.005388977 0.0014417345 0.004943775 0.0051506218 0.009180272 -0.0075472356 -0.005428668 0.0064623333 0.0013423576 -0.0066391225 0.0008783591 0.0027003903 -0.0025289776 -0.004963421 0.0049924683 0.009631416 -0.0073435763 -7.912599e-05 -0.0025523733 -0.0063192695 -0.001368983 -0.005227159 0.009048553 -0.005790704 0.003674939
+is -0.00023357147 0.004226683 0.0021067455 0.009996419 0.0006458492 -0.005461563 -0.0011838758 0.0020920378 -0.0033855627 -0.007853136 -0.005604329 -0.0067612384 0.006366702 0.0039265845 0.008232181 0.0065088123 -0.0061183744 0.002733512 0.008466464 0.0015833755 0.0030677342 0.0058010546 -0.008839754 0.009125629 0.0068226005 0.008512217 -0.0082233 0.0061861346 0.006626654 -0.0013528146 -0.0062799496 0.0053081806 -0.006868758 -0.005337174 0.0035091531 0.008081314 0.008700704 -0.0043939846 -0.0091931205 0.009603682 0.006290027 -0.0039766026 -0.008465367 -0.004691139 -0.0039542373 -0.0032808431 0.0008109401 -0.00030902817 -0.0031103012 -0.005998526 0.009428418 -0.004739384 -0.007274209 0.0076703983 0.0025008747 0.0086274175 -0.004468981 -0.0069012893 0.0009802914 -0.0011801491 -0.009394523 -0.0015968346 0.0030780574 0.006576642 0.0068287384 0.0032347892 -0.0044282703 -0.0018157784 -0.0039494233 0.0057785274 -0.006343468 0.002114367 -0.0013383601 -0.0057999003 -0.007236314 0.0058711045 -0.008345587 -0.00067066104 0.0028193784 0.00773521 -0.007315293 0.003294973 0.009805078 -0.0069755646 -0.003540081 0.005130921 0.005245436 0.0016209023 0.00797557 0.00082546985 0.0018813204 -0.0015988776 -0.008149317 0.0032639706 0.0019852505 -0.008730082 -0.0006569945 7.3046285e-05 -2.6318648e-06 0.008703764
+mitochondrial -0.002508221 -0.0059015388 0.007485539 -0.007257687 -0.008965709 -0.0017888069 -0.008367486 0.00039139786 0.0019467709 -0.0024699308 -0.00644677 -0.00032192905 -0.0010975264 0.0034935323 0.008127049 0.0058537317 0.008440359 -0.0089677265 0.00944024 -0.002368706 0.008696626 0.0023858226 0.0035850583 -0.0095805535 -0.009488111 0.008984071 -0.002896514 0.0028174375 0.0064166263 -0.00029972216 0.00971954 -0.0010352092 -0.009671927 -0.0070548807 -0.0010439103 -0.008674508 0.0074211163 0.0036188734 -0.00874913 0.008480371 0.008929614 0.0058477637 0.0069070626 -0.009568968 0.0004927428 -0.009223568 -0.0036663204 0.00025142074 -0.0002807199 0.0014672013 0.0032786338 0.0021258853 0.005320648 0.0075189634 -0.005886681 0.007957336 0.005991082 0.009785411 0.0046226517 -0.0033269909 -0.0037473391 -0.00062982703 -0.0016548736 0.009871284 0.0011211695 0.00400867 0.0034179776 -0.008850507 0.006720342 0.008190563 -0.0016650181 0.0023356378 -0.0064802184 -0.006126035 0.0082164975 -0.0030429186 0.0067422306 0.001552869 -0.0019822652 0.0030546081 -0.004023311 -0.0017839139 0.0013798403 0.004887597 -0.0014078929 0.0006583137 -0.007930928 0.00949345 -0.008762073 0.007072499 0.0039040898 -0.0069980817 -0.005295161 -0.007937933 -0.0051285303 0.00707022 0.009641066 0.0021544741 0.0006394228 0.009524309

core/NER/word2Vec/word2vec.py

@@ -0,0 +1,436 @@
+'''WORD TO VECTOR'''
+import pandas as pd
+import json
+import gensim
+import spacy
+from core.DefaultPackages import openFile, saveFile
+from core.NER import cleanText
+from gensim.models.keyedvectors import KeyedVectors
+from gensim.test.utils import common_texts
+from gensim.models.word2vec import Word2Vec
+from gensim.scripts.glove2word2vec import glove2word2vec
+from gensim.test.utils import datapath, get_tmpfile
+from gensim.models import Phrases
+from gensim.models.phrases import Phraser
+import sys
+import subprocess
+import os
+# can try multiprocessing to run quicker
+import multiprocessing
+import copy
+sys.setrecursionlimit(1000)
+# creat folder word2Vec
+#! mkdir /content/drive/MyDrive/CollectData/NER/word2Vec
+# create word2vec model
+#model = KeyedVectors.load_word2vec_format('/content/drive/MyDrive/CollectData/NER/word2Vec', binary=True)
+'''Some notes for this model
+sometimes when we do the corpus, there are some adverbs which are unnecessary but might be seen as
+a similar word to the word we are finding, so can we try to preprocess text so that
+we make the corpus more effective and only contains the important words. Then when we
+train the model, the important words will be seen as important. Or
+when we already have the similar list of words, we can remove the words in there
+that are stopwords/unnecessary words.'''
+### For more complex analysis, consider using sentence embedding models like "Doc2Vec" to represent the meaning of entire sentences instead of just individual words
+class word2Vec():
+  def __init__(self, nameFile=None, modelName=None):
+    self.nameFile = nameFile
+    self.modelName = modelName
+    #self.nlp = spacy.load("en_core_web_lg")
+    self.cl = cleanText.cleanGenText()
+  def spacy_similarity(self, word):
+    # when use word2vec, try medium or large is better
+    # maybe try odc similarity?
+    doc = self.nlp(word)
+    for token1 in doc:
+      for token2 in doc:
+        print(token1.text, token2.text, token1.similarity(token2))
+    pass
+  # clean text before transform to corpus
+  def cleanTextBeforeCorpus(self,oriText, doi=None):
+    #cl = cleanText.cleanGenText()
+    #cl = cleanGenText()
+    output = ""
+    alreadyRemoveDoi = False
+    for word in oriText.split(" "):
+      # remove DOI
+      if doi != None and doi in oriText:
+        if alreadyRemoveDoi == False:
+          newWord = self.cl.removeDOI(word,doi)
+          if len(newWord) > 0 and newWord != word:
+            alreadyRemoveDoi = True
+            word = newWord
+      # remove punctuation
+      # split the sticked words
+      #word = cl.splitStickWords(word)
+      # remove punctuation
+      word = self.cl.removePunct(word,True)
+      # remove URL
+      word = self.cl.removeURL(word)
+      # remove HTMLTag
+      word = self.cl.removeHTMLTag(word)
+      # remove tab, white space, newline
+      word = self.cl.removeTabWhiteSpaceNewLine(word)
+      # optional: remove stopwords
+      #word = cl.removeStopWords(word)
+      if len(word)>0:
+        output += word + " "
+    return output
+  def cleanAllTextBeforeCorpus(self, allText, doi=None):
+    cleanOutput = ""
+    remove = "Evaluation Warning: The document was created with Spire.Doc for Python."
+    if len(allText) > 0:
+      corpusText = allText.split("\n\n")
+      for pos in range(len(corpusText)):
+        lines = corpusText[pos]
+        if len(lines) > 0:
+          for line in lines.split("\n"):
+            if remove in line:  line = line.replace(remove, "")
+            clean_text = self.cleanTextBeforeCorpus(line, doi)
+            cleanOutput += clean_text + "\n"
+          cleanOutput += "\n\n"
+    return cleanOutput
+  import urllib.parse, requests
+
+  def tableTransformToCorpusText(self, df, excelFile=None):
+    # PDF, Excel, WordDoc
+    #cl = cleanText.cleanGenText()
+    corpus = {}
+      # PDF or df
+    if excelFile == None:
+      if len(df) > 0:
+        try:
+          for i in range(len(df)):
+            # each new dimension/page is considered to be a sentence which ends with the period.
+            # each new line is a new list, and each new df is a new corpus
+            outputDF = []
+            text = df[i].values.tolist()
+            if len(text) > 0:
+              outputRowDF = self.helperRowTableToCorpus(text)
+              #outputColDF = self.helperColTableToCorpus(text)
+              outputDF.extend(outputRowDF)
+              #outputDF.extend(outputColDF)
+            if len(outputDF) > 0:
+              corpus["corpus" + str(i)] = outputDF
+        except:
+          outputDF = []
+          text = df.values.tolist()
+          if len(text) > 0:
+            outputRowDF = self.helperRowTableToCorpus(text)
+            #outputColDF = self.helperColTableToCorpus(text)
+            outputDF.extend(outputRowDF)
+            #outputDF.extend(outputColDF)
+          if len(outputDF) > 0:
+            corpus["corpus0"] = outputDF
+    else:
+      try:
+          df = pd.ExcelFile(excelFile)
+      except:
+          if excelFile.endswith('.xls'):
+            df = pd.read_excel(excelFile, engine='xlrd')
+          else:
+            df = pd.read_excel(excelFile, engine='openpyxl')    
+      sheetNames = df.sheet_names
+      output = []
+      if len(sheetNames) > 0:
+        for s in range(len(sheetNames)):
+          outputDF = []
+          with pd.ExcelFile(excelFile) as xls:
+            data = pd.read_excel(xls, sheetNames[s])
+          if sheetNames[s] != 'Evaluation Warning':
+            text = data.values.tolist()
+            if len(text) > 0:
+              outputRowDF = self.helperRowTableToCorpus(text)
+              #outputColDF = self.helperColTableToCorpus(text)
+              outputDF.extend(outputRowDF)
+              #outputDF.extend(outputColDF)
+          if len(outputDF) > 0:
+            corpus["corpus" + str(s)] = outputDF
+    return corpus
+  def helperRowTableToCorpus(self, textList):
+    #cl = cleanGenText()
+    #cl = cleanText.cleanGenText()
+    stopWords = ["NaN","Unnamed:","nan"]
+    outputDF = []
+    for line in textList:
+      outputLine = []
+      for words in line:
+        words = str(words)
+        if len(words) > 0:
+          for word in words.split(" "):
+            # remove specific stopwords for table: "NaN", "Unnamed: 0", row index: if the number appears first, it's just a row index; keep "KM1"
+            if str(word) not in stopWords: # remove "NaN", "Unnamed:","nan"
+              #word = cl.splitStickWords(word)
+              word = self.cl.removePunct(word)
+              word = " ".join(self.cl.removeStopWords(word))
+              word = self.cl.removeTabWhiteSpaceNewLine(word)
+              if len(word) > 1:
+                if len(word.split(" ")) > 1:
+                  for x in word.split(" "):
+                    if len(x) > 1 and x.isnumeric()==False:
+                      outputLine.append(x.lower())
+                else:
+                  if word.isnumeric() == False:
+                    outputLine.append(word.lower())
+      if len(outputLine) > 0:
+        outputDF.append(outputLine)
+    return outputDF
+  def helperColTableToCorpus(self, dfList):
+    #cl = cleanGenText()
+    #cl = cleanText.cleanGenText()
+    stopWords = ["NaN","Unnamed:","nan"]
+    outputDF = []
+    # use the first length line as the column ref
+    for pos in range(len(dfList[0])):
+      outputLine = []
+      for line in dfList:
+        if pos < len(line):
+          words = line[pos]
+          words = str(words)
+        else: words = ""
+        if len(words) > 0:
+          for word in words.split(" "):
+            # remove specific stopwords for table: "NaN", "Unnamed: 0", row index: if the number appears first, it's just a row index; keep "KM1"
+            if str(word) not in stopWords: # remove "NaN", "Unnamed:","nan"
+              #word = cl.splitStickWords(word)
+              word = self.cl.removePunct(word)
+              word = " ".join(self.cl.removeStopWords(word))
+              word = self.cl.removeTabWhiteSpaceNewLine(word)
+              if len(word) > 1:
+                if len(word.split(" ")) > 1:
+                  for x in word.split(" "):
+                    if len(x) > 1 and x.isnumeric()==False:
+                      outputLine.append(x.lower())
+                else:
+                  if word.isnumeric() == False:
+                    outputLine.append(word.lower())
+      if len(outputLine) > 0:
+        outputDF.append(outputLine)
+    return outputDF
+  # create a corpus
+  def createCorpusText(self, corpusText):
+    '''ex: "Tom is cat. Jerry is mouse."
+    corpus = [["Tom", "is", "cat"], ["Jerry", "is", "mouse"]]'''
+    # the output should be like this:
+    '''texts = {
+      "Paragraph 1": [["Cat", "is", "an","animal], ["Tom", "is", "cat"]],
+      "Paragraph 2": [["Mouse", "is", "an", "animal"], ["Jerry", "is", "mouse"]]
+    }
+    '''
+    # separate paragraph
+    '''Ex: Cat is an animal. Tom is cat.
+
+    Mouse is an animal.
+    Jerry is mouse.'''
+    texts = {}
+    #cl = cleanText.cleanGenText()
+    #cl = cleanGenText()
+    corpus = corpusText.split("\n\n")
+    for pos in range(len(corpus)):
+      if len(corpus[pos]) > 0:
+        texts["Paragraph "+str(pos)] = []
+        lines = corpus[pos]
+        for line in lines.split("\n"):
+          for l in line.split("."):
+            if len(l) > 0:
+              l = self.cl.removeTabWhiteSpaceNewLine(l)
+              l = l.lower()
+              newL = []
+              for word in l.split(" "):
+                if len(word) > 0:
+                  word = self.cl.removeStopWords(word)
+                  for w in word:
+                    if len(w) > 0 and w.isnumeric()==False:
+                      newL.append(w)
+              if len(newL)>0:
+                texts["Paragraph "+str(pos)].append(newL)
+        if len(texts["Paragraph "+str(pos)]) == 0:
+          del texts["Paragraph "+str(pos)]
+    return texts
+
+  def selectParaForWC(self, corpus):
+    """
+    corpus = [["Tom", "is", "cat"], ["Jerry", "is", "mouse"]]
+    Heuristically determine Word2Vec parameters.
+    """
+    corSize = len(corpus)
+    
+    if corSize == 0:
+        return None, None, None, None, None, None
+
+    # Adjust parameters based on corpus size
+    if corSize < 2000:
+        # Small corpus — need high generalization
+        window = 3
+        vector_size = 100
+        sample = 1e-3
+        negative = 5
+        epochs = 20
+        sg = 1  # Skip-gram preferred for rare words
+    elif corSize < 10000:
+        window = 5
+        vector_size = 150
+        sample = 1e-4
+        negative = 10
+        epochs = 20
+        sg = 1
+    elif corSize < 100000:
+        window = 7
+        vector_size = 200
+        sample = 1e-5
+        negative = 15
+        epochs = 15
+        sg = 1
+    elif corSize < 500000:
+        window = 10
+        vector_size = 250
+        sample = 1e-5
+        negative = 15
+        epochs = 10
+        sg = 0  # CBOW is okay when data is large
+    else:
+        # Very large corpus
+        window = 12
+        vector_size = 300
+        sample = 1e-6
+        negative = 20
+        epochs = 5
+        sg = 0
+
+    return window, vector_size, sample, negative, epochs, sg
+  
+
+  def trainWord2Vec(self,nameFile,modelName,saveFolder,window=None,
+                    vector_size=None,sample=None,negative=None,epochs=None,sg=None):
+    jsonFile = ""
+    jsonFile = openFile.openJsonFile(nameFile) # this is a corpus json file from an article
+    if not jsonFile:
+        print("No corpus to train")
+        return
+    cores = multiprocessing.cpu_count()
+    combinedCorpus = []
+    for key in jsonFile:
+      combinedCorpus.extend(jsonFile[key])
+    # detect phrase before choosing parameters
+    phrases = Phrases(combinedCorpus, min_count=2, threshold=10)
+    bigram = Phraser(phrases)
+    combinedCorpus = [bigram[sent] for sent in combinedCorpus]
+
+    if window==None and vector_size==None and sample==None and negative==None and epochs==None and sg==None:   
+      window, vector_size, sample, negative, epochs, sg = self.selectParaForWC(combinedCorpus)
+    # # min_count=1 ensures all words are included
+    #w2vModel = Word2Vec(vector_size=150, window=10, min_count=1, workers=4)
+    accept = False
+    # add retry limit because if training keeps failing (bad corpus or corrupted input), it’ll keep retrying without limit.
+    retries = 0
+    while not accept and retries < 3:
+      if window!=None and vector_size!=None and sample!=None and negative!=None and epochs!=None and sg!=None:
+        try:
+          w2vModel = Word2Vec(
+                          min_count=1,
+                          window=window,
+                          vector_size=vector_size,
+                          sample=sample,
+                          alpha=0.03,
+                          min_alpha=0.0007,
+                          negative=negative,
+                          workers=cores-1,
+                          epochs = epochs,
+                          sg=sg)
+          w2vModel.build_vocab(combinedCorpus)
+          w2vModel.train(combinedCorpus, total_examples=w2vModel.corpus_count, epochs=epochs)
+          accept = True
+        except Exception as e:
+          print(f"Retry #{retries+1} failed: {e}")
+          retries +=1
+      else:
+        print("no parameter to train")
+        break
+    #w2vModel.build_vocab(combinedCorpus)
+    #w2vModel.train(combinedCorpus, total_examples=w2vModel.corpus_count, epochs=30)
+    #w2vModel.save("/content/drive/MyDrive/CollectData/NER/word2Vec/TestExamples/models/wordVector_"+modelName+".model")
+    #w2vModel.wv.save_word2vec_format("/content/drive/MyDrive/CollectData/NER/word2Vec/TestExamples/models/wordVector_"+modelName+".txt")
+    w2vModel.save(saveFolder+"/"+modelName+".model")
+    w2vModel.wv.save_word2vec_format(saveFolder+"/"+modelName+".txt")
+    print("done w2v")
+    #return combinedCorpus
+  def updateWord2Vec(self, modelPath, newCorpus, saveFolder=None):
+    if not newCorpus:
+        raise ValueError("New corpus is empty!")
+
+    model = Word2Vec.load(modelPath)
+
+    # Phrase detection on new data
+    phrases = Phrases(newCorpus, min_count=2, threshold=10)
+    bigram = Phraser(phrases)
+    newCorpus = [bigram[sent] for sent in newCorpus]
+
+    # Update vocab & retrain
+    model.build_vocab(newCorpus, update=True)
+    model.train(newCorpus, total_examples=len(newCorpus), epochs=model.epochs)
+
+  def genSimilar(self,word,modelFile,n=10, cos_thres=0.7):
+    # might not be a meaningful keyword
+    #stopWords = ["show"]
+    # same word but just plural nouns, tense
+    simWords = [word+"s",word+"es",word+"ing",word+"ed"]
+    model = KeyedVectors.load_word2vec_format(modelFile, binary = False) # model file in format txt
+    results = model.most_similar(positive=[word],topn=n)
+    #removeIndex = []
+    #currN = copy.deepcopy(n)
+    '''for r in range(len(results)):
+      if len(results[r][0]) < 2:
+        removeIndex.append(results[r])
+      # remove the same word but just plural and singular noun and lower than the cos_thres
+      elif results[r][0] == word:
+        removeIndex.append(results[r])
+      elif results[r][0] in simWords or float(results[r][1]) < cos_thres or results[r][0] in stopWords:
+        removeIndex.append(results[r])
+    for rem in removeIndex:
+      results.remove(rem)
+    while len(results)!=n and len(results) != 0:
+      moreNewResult = model.most_similar(positive=[word],topn=currN+1)[-1]
+      if moreNewResult not in results and len(moreNewResult[0])>1:
+        if moreNewResult[0] not in stopWords and results[0] != word:
+          results.append(moreNewResult)
+      currN +=1'''
+    return results
+  # add more data to existing word2vec model
+  def updateWord2Vec(self, modelPath, newCorpus, saveFolder=None):
+    if not newCorpus:
+        raise ValueError("New corpus is empty!")
+
+    model = Word2Vec.load(modelPath)
+
+    # Phrase detection on new data
+    phrases = Phrases(newCorpus, min_count=2, threshold=10)
+    bigram = Phraser(phrases)
+    newCorpus = [bigram[sent] for sent in newCorpus]
+
+    # Update vocab & retrain
+    model.build_vocab(newCorpus, update=True)
+    model.train(newCorpus, total_examples=len(newCorpus), epochs=model.epochs)
+
+    # Save updated model
+    if saveFolder:
+        os.makedirs(saveFolder, exist_ok=True)
+        name = os.path.basename(modelPath).replace(".model", "_updated.model")
+        model.save(f"{saveFolder}/{name}")
+        print(f"🔁 Model updated and saved to {saveFolder}/{name}")
+    else:
+        model.save(modelPath)
+        print(f"🔁 Model updated and overwritten at {modelPath}")
+  
+  # adding our model into spacy
+  # this deals with command line; but instead of using it, we write python script to run command line
+  def loadWordVec(self,modelName,wordVec):
+    # modelName is the name you want to save into spacy
+    # wordVec is the trained word2vec in txt format
+    subprocess.run([sys.executable,
+                    "-m",
+                    "spacy",
+                    "init-model",
+                    "en",
+                    modelName, # this modelName comes from the saved modelName of function trainWord2Vec
+                    "--vectors-loc",
+                    wordVec])
+    print("done")

core/data_preprocess.py

@@ -0,0 +1,744 @@
+import re, os, json, tempfile, subprocess, nltk
+
+#import streamlit as st
+from Bio import Entrez
+from docx import Document
+import fitz
+import spacy
+from spacy.cli import download
+
+import core.model
+import core.pipeline
+from core.drive_utils import upload_file_to_drive   
+from core.NER.PDF import pdf
+from core.NER.WordDoc import wordDoc
+from core.NER.html import extractHTML
+from core.NER.word2Vec import word2vec
+#from transformers import pipeline
+import urllib.parse, requests
+from pathlib import Path
+import pandas as pd
+
+
+nltk.download('punkt_tab')
+def download_excel_file(url, save_path="temp.xlsx"):
+    if "view.officeapps.live.com" in url:
+        parsed_url = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
+        real_url = urllib.parse.unquote(parsed_url["src"][0])
+        response = requests.get(real_url)
+        with open(save_path, "wb") as f:
+            f.write(response.content)
+        return save_path
+    elif url.startswith("http") and (url.endswith(".xls") or url.endswith(".xlsx")):
+        response = requests.get(url)
+        response.raise_for_status()  # Raises error if download fails
+        with open(save_path, "wb") as f:
+            f.write(response.content)
+            print(len(response.content))
+        return save_path
+    else:
+        print("URL must point directly to an .xls or .xlsx file\n or it already downloaded.")
+        return url
+def extract_text(link,saveFolder):
+  try:
+      text = ""
+      name = link.split("/")[-1]
+      print("name: ", name)  
+      #file_path = Path(saveFolder) / name
+      local_temp_path = os.path.join(tempfile.gettempdir(), name)
+      print("this is local temp path: ", local_temp_path)  
+      if os.path.exists(local_temp_path):
+        input_to_class = local_temp_path
+        print("exist")  
+      else:
+        #input_to_class = link  # Let the class handle downloading  
+        # 1. Check if file exists in shared Google Drive folder
+        file_id = pipeline.find_drive_file(name, saveFolder)
+        if file_id:
+            print("📥 Downloading from Google Drive...")
+            pipeline.download_file_from_drive(name, saveFolder, local_temp_path)
+        else:
+            print("🌐 Downloading from web link...")
+            response = requests.get(link)
+            with open(local_temp_path, 'wb') as f:
+                f.write(response.content)
+            print("✅ Saved locally.")
+    
+            # 2. Upload to Drive so it's available for later
+            pipeline.upload_file_to_drive(local_temp_path, name, saveFolder)
+    
+        input_to_class = local_temp_path
+        print(input_to_class)  
+      # pipeline.download_file_from_drive(name, saveFolder, local_temp_path)  
+      # pdf
+      if link.endswith(".pdf"):
+        # if file_path.is_file():
+        #   link = saveFolder + "/" + name
+        #   print("File exists.")
+        #p = pdf.PDF(local_temp_path, saveFolder)
+        print("inside pdf and input to class: ", input_to_class)  
+        print("save folder in extract text: ", saveFolder)  
+        p = pdf.PDF(input_to_class, saveFolder)  
+        #p = pdf.PDF(link,saveFolder)
+        #text = p.extractTextWithPDFReader()
+        text = p.extractText()  
+        print("text from pdf:")
+        print(text)  
+        #text_exclude_table = p.extract_text_excluding_tables()
+      # worddoc
+      elif link.endswith(".doc") or link.endswith(".docx"):
+        #d = wordDoc.wordDoc(local_temp_path,saveFolder)
+        d = wordDoc.wordDoc(input_to_class,saveFolder)  
+        text = d.extractTextByPage()
+      # html
+      else:  
+        if link.split(".")[-1].lower() not in "xlsx":
+            if "http" in link or "html" in link:
+              print("html link: ", link)  
+              html = extractHTML.HTML("",link)
+              text = html.getListSection() # the text already clean
+              print("text html: ")
+              print(text)  
+      # Cleanup: delete the local temp file
+      if name:
+          if os.path.exists(local_temp_path):
+            os.remove(local_temp_path)
+            print(f"🧹 Deleted local temp file: {local_temp_path}")   
+      print("done extract text")        
+  except:
+      text = ""
+  return text
+
+def extract_table(link,saveFolder):
+  try:  
+      table = []
+      name = link.split("/")[-1]
+      #file_path = Path(saveFolder) / name
+      local_temp_path = os.path.join(tempfile.gettempdir(), name)
+      if os.path.exists(local_temp_path):
+        input_to_class = local_temp_path
+        print("exist")  
+      else:
+        #input_to_class = link  # Let the class handle downloading  
+        # 1. Check if file exists in shared Google Drive folder
+        file_id = pipeline.find_drive_file(name, saveFolder)
+        if file_id:
+            print("📥 Downloading from Google Drive...")
+            pipeline.download_file_from_drive(name, saveFolder, local_temp_path)
+        else:
+            print("🌐 Downloading from web link...")
+            response = requests.get(link)
+            with open(local_temp_path, 'wb') as f:
+                f.write(response.content)
+            print("✅ Saved locally.")
+    
+            # 2. Upload to Drive so it's available for later
+            pipeline.upload_file_to_drive(local_temp_path, name, saveFolder)
+    
+        input_to_class = local_temp_path
+        print(input_to_class)
+      #pipeline.download_file_from_drive(name, saveFolder, local_temp_path)
+      # pdf
+      if link.endswith(".pdf"):
+        # if file_path.is_file():
+        #   link = saveFolder + "/" + name
+        #   print("File exists.")
+        #p = pdf.PDF(local_temp_path,saveFolder)
+        p = pdf.PDF(input_to_class,saveFolder)  
+        table = p.extractTable()
+      # worddoc
+      elif link.endswith(".doc") or link.endswith(".docx"):
+        #d = wordDoc.wordDoc(local_temp_path,saveFolder)
+        d = wordDoc.wordDoc(input_to_class,saveFolder)  
+        table = d.extractTableAsList()
+      # excel
+      elif link.split(".")[-1].lower() in "xlsx":
+        # download excel file if it not downloaded yet
+        savePath = saveFolder +"/"+ link.split("/")[-1]
+        excelPath = download_excel_file(link, savePath)
+        try:
+            #xls = pd.ExcelFile(excelPath)
+            xls = pd.ExcelFile(local_temp_path)
+            table_list = []
+            for sheet_name in xls.sheet_names:
+                df = pd.read_excel(xls, sheet_name=sheet_name)
+                cleaned_table = df.fillna("").astype(str).values.tolist()
+                table_list.append(cleaned_table)
+            table = table_list
+        except Exception as e:
+            print("❌ Failed to extract tables from Excel:", e)
+      # html
+      elif "http" in link or "html" in link:
+        html = extractHTML.HTML("",link)
+        table = html.extractTable() # table is a list
+      table = clean_tables_format(table)
+      # Cleanup: delete the local temp file
+      if os.path.exists(local_temp_path):
+        os.remove(local_temp_path)
+        print(f"🧹 Deleted local temp file: {local_temp_path}")
+  except:
+      table = []
+  return table
+
+def clean_tables_format(tables):
+    """
+    Ensures all tables are in consistent format: List[List[List[str]]]
+    Cleans by:
+    - Removing empty strings and rows
+    - Converting all cells to strings
+    - Handling DataFrames and list-of-lists
+    """
+    cleaned = []
+    if tables:
+      for table in tables:
+          standardized = []
+
+          # Case 1: Pandas DataFrame
+          if isinstance(table, pd.DataFrame):
+              table = table.fillna("").astype(str).values.tolist()
+
+          # Case 2: List of Lists
+          if isinstance(table, list) and all(isinstance(row, list) for row in table):
+              for row in table:
+                  filtered_row = [str(cell).strip() for cell in row if str(cell).strip()]
+                  if filtered_row:
+                      standardized.append(filtered_row)
+
+          if standardized:
+              cleaned.append(standardized)
+
+    return cleaned
+
+def normalize_text_for_comparison(s: str) -> str:
+    """
+    Normalizes text for robust comparison by:
+    1. Converting to lowercase.
+    2. Replacing all types of newlines with a single consistent newline (\n).
+    3. Removing extra spaces (e.g., multiple spaces, leading/trailing spaces on lines).
+    4. Stripping leading/trailing whitespace from the entire string.
+    """
+    s = s.lower()
+    s = s.replace('\r\n', '\n') # Handle Windows newlines
+    s = s.replace('\r', '\n')   # Handle Mac classic newlines
+    
+    # Replace sequences of whitespace (including multiple newlines) with a single space
+    # This might be too aggressive if you need to preserve paragraph breaks,
+    # but good for exact word-sequence matching.
+    s = re.sub(r'\s+', ' ', s) 
+    
+    return s.strip()
+def merge_text_and_tables(text, tables, max_tokens=12000, keep_tables=True, tokenizer="cl100k_base", accession_id=None, isolate=None):
+    """
+    Merge cleaned text and table into one string for LLM input.
+    - Avoids duplicating tables already in text
+    - Extracts only relevant rows from large tables
+    - Skips or saves oversized tables
+    """
+    import importlib
+    json = importlib.import_module("json")
+
+    def estimate_tokens(text_str):
+        try:
+            enc = tiktoken.get_encoding(tokenizer)
+            return len(enc.encode(text_str))
+        except:
+            return len(text_str) // 4  # Fallback estimate
+
+    def is_table_relevant(table, keywords, accession_id=None):
+        flat = " ".join(" ".join(row).lower() for row in table)
+        if accession_id and accession_id.lower() in flat:
+            return True    
+        return any(kw.lower() in flat for kw in keywords)
+    preview, preview1 = "",""    
+    llm_input = "## Document Text\n" + text.strip() + "\n"
+    clean_text = normalize_text_for_comparison(text)
+
+    if tables:
+        for idx, table in enumerate(tables):
+          keywords = ["province","district","region","village","location", "country", "region", "origin", "ancient", "modern"]
+          if accession_id:  keywords += [accession_id.lower()]
+          if isolate: keywords += [isolate.lower()]
+          if is_table_relevant(table, keywords, accession_id):
+            if len(table) > 0:
+              for tab in table:
+                preview = " ".join(tab) if tab else ""
+                preview1 = "\n".join(tab) if tab else ""
+                clean_preview = normalize_text_for_comparison(preview)
+                clean_preview1 = normalize_text_for_comparison(preview1)
+                if clean_preview not in clean_text:
+                  if clean_preview1 not in clean_text:
+                    table_str = json.dumps([tab], indent=2)
+                    llm_input += f"## Table {idx+1}\n{table_str}\n"
+    return llm_input.strip()
+
+def preprocess_document(link, saveFolder, accession=None, isolate=None):
+    try:
+      text = extract_text(link, saveFolder)
+      print("text and link")
+      print(link)
+      print(text)  
+    except: text = ""  
+    try: 
+      tables = extract_table(link, saveFolder)
+    except: tables = []  
+    if accession: accession = accession
+    if isolate: isolate = isolate
+    try:
+      final_input = merge_text_and_tables(text, tables, max_tokens=12000, accession_id=accession, isolate=isolate)
+    except: final_input = ""
+    return text, tables, final_input
+
+def extract_sentences(text):
+    sentences = re.split(r'(?<=[.!?])\s+', text)
+    return [s.strip() for s in sentences if s.strip()]
+
+def is_irrelevant_number_sequence(text):
+    if re.search(r'\b[A-Z]{2,}\d+\b|\b[A-Za-z]+\s+\d+\b', text, re.IGNORECASE):
+        return False
+    word_count = len(re.findall(r'\b[A-Za-z]{2,}\b', text))
+    number_count = len(re.findall(r'\b\d[\d\.]*\b', text))
+    total_tokens = len(re.findall(r'\S+', text))
+    if total_tokens > 0 and (word_count / total_tokens < 0.2) and (number_count / total_tokens > 0.5):
+        return True
+    elif re.fullmatch(r'(\d+(\.\d+)?\s*)+', text.strip()):
+        return True
+    return False
+
+def remove_isolated_single_digits(sentence):
+    tokens = sentence.split()
+    filtered_tokens = []
+    for token in tokens:
+        if token == '0' or token == '1':
+            pass
+        else:
+            filtered_tokens.append(token)
+    return ' '.join(filtered_tokens).strip()
+
+def get_contextual_sentences_BFS(text_content, keyword, depth=2):
+    def extract_codes(sentence):
+    # Match codes like 'A1YU101', 'KM1', 'MO6' — at least 2 letters + numbers
+      return [code for code in re.findall(r'\b[A-Z]{2,}[0-9]+\b', sentence, re.IGNORECASE)]
+    sentences = extract_sentences(text_content)
+    relevant_sentences = set()
+    initial_keywords = set()
+
+    # Define a regex to capture codes like A1YU101 or KM1
+    # This pattern looks for an alphanumeric sequence followed by digits at the end of the string
+    code_pattern = re.compile(r'([A-Z0-9]+?)(\d+)$', re.IGNORECASE)
+
+    # Attempt to parse the keyword into its prefix and numerical part using re.search
+    keyword_match = code_pattern.search(keyword)
+
+    keyword_prefix = None
+    keyword_num = None
+
+    if keyword_match:
+        keyword_prefix = keyword_match.group(1).lower()
+        keyword_num = int(keyword_match.group(2))
+
+    for sentence in sentences:
+        sentence_added = False
+
+        # 1. Check for exact match of the keyword
+        if re.search(r'\b' + re.escape(keyword) + r'\b', sentence, re.IGNORECASE):
+            relevant_sentences.add(sentence.strip())
+            initial_keywords.add(keyword.lower())
+            sentence_added = True
+
+        # 2. Check for range patterns (e.g., A1YU101-A1YU137)
+        # The range pattern should be broad enough to capture the full code string within the range.
+        range_matches = re.finditer(r'([A-Z0-9]+-\d+)', sentence, re.IGNORECASE) # More specific range pattern if needed, or rely on full code pattern below
+        range_matches = re.finditer(r'([A-Z0-9]+\d+)-([A-Z0-9]+\d+)', sentence, re.IGNORECASE) # This is the more robust range pattern
+
+        for r_match in range_matches:
+            start_code_str = r_match.group(1)
+            end_code_str = r_match.group(2)
+
+            # CRITICAL FIX: Use code_pattern.search for start_match and end_match
+            start_match = code_pattern.search(start_code_str)
+            end_match = code_pattern.search(end_code_str)
+
+            if keyword_prefix and keyword_num is not None and start_match and end_match:
+                start_prefix = start_match.group(1).lower()
+                end_prefix = end_match.group(1).lower()
+                start_num = int(start_match.group(2))
+                end_num = int(end_match.group(2))
+
+                # Check if the keyword's prefix matches and its number is within the range
+                if keyword_prefix == start_prefix and \
+                   keyword_prefix == end_prefix and \
+                   start_num <= keyword_num <= end_num:
+                    relevant_sentences.add(sentence.strip())
+                    initial_keywords.add(start_code_str.lower())
+                    initial_keywords.add(end_code_str.lower())
+                    sentence_added = True
+                    break # Only need to find one matching range per sentence
+
+        # 3. If the sentence was added due to exact match or range, add all its alphanumeric codes
+        #    to initial_keywords to ensure graph traversal from related terms.
+        if sentence_added:
+          for word in extract_codes(sentence):
+            initial_keywords.add(word.lower())
+
+
+    # Build word_to_sentences mapping for all sentences
+    word_to_sentences = {}
+    for sent in sentences:
+      codes_in_sent = set(extract_codes(sent))
+      for code in codes_in_sent:
+          word_to_sentences.setdefault(code.lower(), set()).add(sent.strip())
+
+
+    # Build the graph
+    graph = {}
+    for sent in sentences:
+      codes = set(extract_codes(sent))
+      for word1 in codes:
+          word1_lower = word1.lower()
+          graph.setdefault(word1_lower, set())
+          for word2 in codes:
+              word2_lower = word2.lower()
+              if word1_lower != word2_lower:
+                  graph[word1_lower].add(word2_lower)
+
+
+    # Perform BFS/graph traversal
+    queue = [(k, 0) for k in initial_keywords if k in word_to_sentences]
+    visited_words = set(initial_keywords)
+
+    while queue:
+        current_word, level = queue.pop(0)
+        if level >= depth:
+            continue
+
+        relevant_sentences.update(word_to_sentences.get(current_word, []))
+
+        for neighbor in graph.get(current_word, []):
+            if neighbor not in visited_words:
+                visited_words.add(neighbor)
+                queue.append((neighbor, level + 1))
+
+    final_sentences = set()
+    for sentence in relevant_sentences:
+        if not is_irrelevant_number_sequence(sentence):
+            processed_sentence = remove_isolated_single_digits(sentence)
+            if processed_sentence:
+                final_sentences.add(processed_sentence)
+
+    return "\n".join(sorted(list(final_sentences)))
+
+
+
+def get_contextual_sentences_DFS(text_content, keyword, depth=2):
+    sentences = extract_sentences(text_content)
+
+    # Build word-to-sentences mapping
+    word_to_sentences = {}
+    for sent in sentences:
+        words_in_sent = set(re.findall(r'\b[A-Za-z0-9\-_\/]+\b', sent))
+        for word in words_in_sent:
+            word_to_sentences.setdefault(word.lower(), set()).add(sent.strip())
+
+    # Function to extract codes in a sentence
+    def extract_codes(sentence):
+      # Only codes like 'KSK1', 'MG272794', not pure numbers
+      return [code for code in re.findall(r'\b[A-Z]{2,}[0-9]+\b', sentence, re.IGNORECASE)]
+
+    # DFS with priority based on distance to keyword and early stop if country found
+    def dfs_traverse(current_word, current_depth, max_depth, visited_words, collected_sentences, parent_sentence=None):
+        country = "unknown"
+        if current_depth > max_depth:
+            return country, False
+
+        if current_word not in word_to_sentences:
+            return country, False
+
+        for sentence in word_to_sentences[current_word]:
+            if sentence == parent_sentence:
+                continue  # avoid reusing the same sentence
+
+            collected_sentences.add(sentence)
+
+            #print("current_word:", current_word)
+            small_sen = extract_context(sentence, current_word, int(len(sentence) / 4))
+            #print(small_sen)
+            country = model.get_country_from_text(small_sen)
+            #print("small context country:", country)
+            if country.lower() != "unknown":
+                return country, True
+            else:
+                country = model.get_country_from_text(sentence)
+                #print("full sentence country:", country)
+                if country.lower() != "unknown":
+                    return country, True
+
+            codes_in_sentence = extract_codes(sentence)
+            idx = next((i for i, code in enumerate(codes_in_sentence) if code.lower() == current_word.lower()), None)
+            if idx is None:
+                continue
+
+            sorted_children = sorted(
+                [code for code in codes_in_sentence if code.lower() not in visited_words],
+                key=lambda x: (abs(codes_in_sentence.index(x) - idx),
+                               0 if codes_in_sentence.index(x) > idx else 1)
+            )
+
+            #print("sorted_children:", sorted_children)
+            for child in sorted_children:
+                child_lower = child.lower()
+                if child_lower not in visited_words:
+                    visited_words.add(child_lower)
+                    country, should_stop = dfs_traverse(
+                        child_lower, current_depth + 1, max_depth,
+                        visited_words, collected_sentences, parent_sentence=sentence
+                    )
+                    if should_stop:
+                        return country, True
+
+        return country, False
+
+    # Begin DFS
+    collected_sentences = set()
+    visited_words = set([keyword.lower()])
+    country, status = dfs_traverse(keyword.lower(), 0, depth, visited_words, collected_sentences)
+
+    # Filter irrelevant sentences
+    final_sentences = set()
+    for sentence in collected_sentences:
+        if not is_irrelevant_number_sequence(sentence):
+            processed = remove_isolated_single_digits(sentence)
+            if processed:
+                final_sentences.add(processed)
+    if not final_sentences:
+      return country, text_content
+    return country, "\n".join(sorted(list(final_sentences)))
+
+# Helper function for normalizing text for overlap comparison
+def normalize_for_overlap(s: str) -> str:
+    s = re.sub(r'[^a-zA-Z0-9\s]', ' ', s).lower()
+    s = re.sub(r'\s+', ' ', s).strip()
+    return s
+
+def merge_texts_skipping_overlap(text1: str, text2: str) -> str:
+    if not text1: return text2
+    if not text2: return text1
+
+    # Case 1: text2 is fully contained in text1 or vice-versa
+    if text2 in text1:
+        return text1
+    if text1 in text2:
+        return text2
+
+    # --- Option 1: Original behavior (suffix of text1, prefix of text2) ---
+    # This is what your function was primarily designed for.
+    # It looks for the overlap at the "junction" of text1 and text2.
+    
+    max_junction_overlap = 0
+    for i in range(min(len(text1), len(text2)), 0, -1):
+        suffix1 = text1[-i:]
+        prefix2 = text2[:i]
+        # Prioritize exact match, then normalized match
+        if suffix1 == prefix2:
+            max_junction_overlap = i
+            break
+        elif normalize_for_overlap(suffix1) == normalize_for_overlap(prefix2):
+            max_junction_overlap = i
+            break # Take the first (longest) normalized match
+
+    if max_junction_overlap > 0:
+        merged_text = text1 + text2[max_junction_overlap:]
+        return re.sub(r'\s+', ' ', merged_text).strip()
+
+    # --- Option 2: Longest Common Prefix (for cases like "Hi, I am Vy.") ---
+    # This addresses your specific test case where the overlap is at the very beginning of both strings.
+    # This is often used when trying to deduplicate content that shares a common start.
+
+    longest_common_prefix_len = 0
+    min_len = min(len(text1), len(text2))
+    for i in range(min_len):
+        if text1[i] == text2[i]:
+            longest_common_prefix_len = i + 1
+        else:
+            break
+    
+    # If a common prefix is found AND it's a significant portion (e.g., more than a few chars)
+    # AND the remaining parts are distinct, then apply this merge.
+    # This is a heuristic and might need fine-tuning.
+    if longest_common_prefix_len > 0 and \
+       text1[longest_common_prefix_len:].strip() and \
+       text2[longest_common_prefix_len:].strip():
+
+        # Only merge this way if the remaining parts are not empty (i.e., not exact duplicates)
+        # For "Hi, I am Vy. Nice to meet you." and "Hi, I am Vy. Goodbye Vy."
+        # common prefix is "Hi, I am Vy."
+        # Remaining text1: " Nice to meet you."
+        # Remaining text2: " Goodbye Vy."
+        # So we merge common_prefix + remaining_text1 + remaining_text2
+        
+        common_prefix_str = text1[:longest_common_prefix_len]
+        remainder_text1 = text1[longest_common_prefix_len:]
+        remainder_text2 = text2[longest_common_prefix_len:]
+        
+        merged_text = common_prefix_str + remainder_text1 + remainder_text2
+        return re.sub(r'\s+', ' ', merged_text).strip()
+
+
+    # If neither specific overlap type is found, just concatenate
+    merged_text = text1 + text2
+    return re.sub(r'\s+', ' ', merged_text).strip()
+ 
+# def save_text_to_docx(text_content: str, file_path: str):
+#     """
+#     Saves a given text string into a .docx file.
+
+#     Args:
+#         text_content (str): The text string to save.
+#         file_path (str): The full path including the filename where the .docx file will be saved.
+#                          Example: '/content/drive/MyDrive/CollectData/Examples/test/SEA_1234/merged_document.docx'
+#     """
+#     try:
+#         document = Document()
+
+#         # Add the entire text as a single paragraph, or split by newlines for multiple paragraphs
+#         for paragraph_text in text_content.split('\n'):
+#             document.add_paragraph(paragraph_text)
+
+#         document.save(file_path)
+#         print(f"Text successfully saved to '{file_path}'")
+#     except Exception as e:
+#         print(f"Error saving text to docx file: {e}") 
+# def save_text_to_docx(text_content: str, filename: str, drive_folder_id: str):
+#     """
+#     Saves a given text string into a .docx file locally, then uploads to Google Drive.
+
+#     Args:
+#         text_content (str): The text string to save.
+#         filename (str): The target .docx file name, e.g. 'BRU18_merged_document.docx'.
+#         drive_folder_id (str): Google Drive folder ID where to upload the file.
+#     """
+#     try:
+#         # ✅ Save to temporary local path first
+#         print("file name: ", filename)
+#         print("length text content: ", len(text_content))
+#         local_path = os.path.join(tempfile.gettempdir(), filename)
+#         document = Document()
+#         for paragraph_text in text_content.split('\n'):
+#             document.add_paragraph(paragraph_text)
+#         document.save(local_path)
+#         print(f"✅ Text saved locally to: {local_path}")
+
+#         # ✅ Upload to Drive
+#         pipeline.upload_file_to_drive(local_path, filename, drive_folder_id)
+#         print(f"✅ Uploaded '{filename}' to Google Drive folder ID: {drive_folder_id}")
+
+#     except Exception as e:
+#         print(f"❌ Error saving or uploading DOCX: {e}")
+def save_text_to_docx(text_content: str, full_local_path: str):
+    document = Document()
+    for paragraph_text in text_content.split('\n'):
+        document.add_paragraph(paragraph_text)
+    document.save(full_local_path)
+    print(f"✅ Saved DOCX locally: {full_local_path}")
+
+
+
+'''2 scenerios:
+- quick look then found then deepdive and directly get location then stop
+- quick look then found then deepdive but not find location then hold the related words then 
+look another files iteratively for each related word and find location and stop'''
+def extract_context(text, keyword, window=500):
+    # firstly try accession number
+    code_pattern = re.compile(r'([A-Z0-9]+?)(\d+)$', re.IGNORECASE)
+
+    # Attempt to parse the keyword into its prefix and numerical part using re.search
+    keyword_match = code_pattern.search(keyword)
+
+    keyword_prefix = None
+    keyword_num = None
+
+    if keyword_match:
+        keyword_prefix = keyword_match.group(1).lower()
+        keyword_num = int(keyword_match.group(2))
+    text = text.lower()    
+    idx = text.find(keyword.lower())
+    if idx == -1:
+      if keyword_prefix:
+        idx = text.find(keyword_prefix)
+      if idx == -1:
+        return "Sample ID not found."
+      return text[max(0, idx-window): idx+window]  
+    return text[max(0, idx-window): idx+window]
+def process_inputToken(filePaths, saveLinkFolder,accession=None, isolate=None):
+  cache = {}
+  country = "unknown"
+  output = ""
+  tem_output, small_output = "",""
+  keyword_appear = (False,"")
+  keywords = []
+  if isolate: keywords.append(isolate)
+  if accession: keywords.append(accession)
+  for f in filePaths:
+    # scenerio 1: direct location: truncate the context and then use qa model?
+    if keywords:
+      for keyword in keywords:
+        text, tables, final_input = preprocess_document(f,saveLinkFolder, isolate=keyword)
+        if keyword in final_input:
+          context = extract_context(final_input, keyword)
+          # quick look if country already in context and if yes then return
+          country = model.get_country_from_text(context)
+          if country != "unknown":
+            return country, context, final_input
+          else:
+            country = model.get_country_from_text(final_input)  
+            if country != "unknown":
+              return country, context, final_input
+            else: # might be cross-ref
+              keyword_appear = (True, f)
+              cache[f] = context
+              small_output = merge_texts_skipping_overlap(output, context) + "\n"
+              chunkBFS = get_contextual_sentences_BFS(small_output, keyword)
+              countryBFS = model.get_country_from_text(chunkBFS)
+              countryDFS, chunkDFS = get_contextual_sentences_DFS(output, keyword)
+              output = merge_texts_skipping_overlap(output, final_input)
+              if countryDFS != "unknown" and countryBFS != "unknown":
+                if len(chunkDFS) <= len(chunkBFS):
+                  return countryDFS, chunkDFS, output
+                else:
+                  return countryBFS, chunkBFS, output
+              else:        
+                if countryDFS != "unknown":  
+                  return countryDFS, chunkDFS, output
+                if countryBFS != "unknown":
+                  return countryBFS, chunkBFS, output
+        else:
+        # scenerio 2: 
+          '''cross-ref: ex: A1YU101 keyword in file 2 which includes KM1 but KM1 in file 1 
+          but if we look at file 1 first then maybe we can have lookup dict which country 
+          such as Thailand as the key and its re''' 
+          cache[f] = final_input
+          if keyword_appear[0] == True:
+            for c in cache:
+              if c!=keyword_appear[1]:
+                if cache[c].lower() not in output.lower():
+                  output = merge_texts_skipping_overlap(output, cache[c]) + "\n"
+                  chunkBFS = get_contextual_sentences_BFS(output, keyword)
+                  countryBFS = model.get_country_from_text(chunkBFS)
+                  countryDFS, chunkDFS = get_contextual_sentences_DFS(output, keyword)
+                  if countryDFS != "unknown" and countryBFS != "unknown":
+                    if len(chunkDFS) <= len(chunkBFS):
+                      return countryDFS, chunkDFS, output
+                    else:
+                      return countryBFS, chunkBFS, output
+                  else:        
+                    if countryDFS != "unknown":  
+                      return countryDFS, chunkDFS, output
+                    if countryBFS != "unknown":
+                      return countryBFS, chunkBFS, output
+          else:
+            if cache[f].lower() not in output.lower():
+              output = merge_texts_skipping_overlap(output, cache[f]) + "\n"          
+  if len(output) == 0 or keyword_appear[0]==False:
+    for c in cache:
+      if cache[c].lower() not in output.lower():
+        output = merge_texts_skipping_overlap(output, cache[c]) + "\n"
+  return country, "", output               

core/drive_utils.py

@@ -0,0 +1,138 @@
+import google.generativeai as genai
+# Google Drive (optional)
+from google.oauth2.service_account import Credentials
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload
+import gspread
+from oauth2client.service_account import ServiceAccountCredentials
+
+import os, io, time, re, json
+
+#––– Authentication setup –––
+GDRIVE_PARENT_FOLDER_NAME = "mtDNA-Location-Classifier"
+GDRIVE_DATA_FOLDER_NAME = os.environ["GDRIVE_DATA_FOLDER_NAME"]
+GCP_CREDS_DICT = json.loads(os.environ["GCP_CREDS_JSON"])  # from HF secrets
+GDRIVE_CREDS = Credentials.from_service_account_info(GCP_CREDS_DICT, scopes=["https://www.googleapis.com/auth/drive"])
+drive_service = build("drive", "v3", credentials=GDRIVE_CREDS)
+
+def get_or_create_drive_folder(name, parent_id=None):
+    query = f"name='{name}' and mimeType='application/vnd.google-apps.folder'"
+    if parent_id:
+        query += f" and '{parent_id}' in parents"
+    results = drive_service.files().list(q=query, spaces='drive', fields="files(id, name)").execute()
+    items = results.get("files", [])
+    if items:
+        return items[0]["id"]
+    file_metadata = {
+        "name": name,
+        "mimeType": "application/vnd.google-apps.folder"
+    }
+    if parent_id:
+        file_metadata["parents"] = [parent_id]
+    file = drive_service.files().create(body=file_metadata, fields="id").execute()
+    return file["id"]
+# def find_drive_file(filename, parent_id):
+#     """
+#     Checks if a file with the given name exists inside the specified Google Drive folder.
+#     Returns the file ID if found, else None.
+#     """
+#     query = f"'{parent_id}' in parents and name = '{filename}' and trashed = false"
+#     results = drive_service.files().list(q=query, spaces='drive', fields='files(id, name)', pageSize=1).execute()
+#     files = results.get('files', [])
+#     if files:
+#         return files[0]["id"]
+#     return None
+
+def find_drive_file(filename, parent_id):
+    """
+    Checks if a file with the given name exists inside the specified Google Drive folder.
+    Returns the file ID if found, else None.
+    """
+    try:
+        print(f"🔍 Searching for '{filename}' in folder: {parent_id}")
+        query = f"'{parent_id}' in parents and name = '{filename}' and trashed = false"
+        results = drive_service.files().list(
+            q=query,
+            spaces='drive',
+            fields='files(id, name)',
+            pageSize=1
+        ).execute()
+        files = results.get('files', [])
+        if files:
+            print(f"✅ Found file: {files[0]['name']} with ID: {files[0]['id']}")
+            return files[0]["id"]
+        else:
+            print("⚠️ File not found.")
+            return None
+    except Exception as e:
+        print(f"❌ Error during find_drive_file: {e}")
+        return None
+
+
+
+# def upload_file_to_drive(local_path, remote_name, folder_id):
+#     file_metadata = {"name": remote_name, "parents": [folder_id]}
+#     media = MediaFileUpload(local_path, resumable=True)
+#     existing = drive_service.files().list(q=f"name='{remote_name}' and '{folder_id}' in parents", fields="files(id)").execute().get("files", [])
+#     if existing:
+#         drive_service.files().delete(fileId=existing[0]["id"]).execute()
+#     file = drive_service.files().create(body=file_metadata, media_body=media, fields="id").execute()
+#     result = drive_service.files().list(q=f"name='{remote_name}' and '{folder_id}' in parents", fields="files(id)").execute()
+#     if not result.get("files"):
+#         print(f"❌ Upload failed: File '{remote_name}' not found in folder after upload.")
+#     else:
+#         print(f"✅ Verified upload: {remote_name}")
+#     return file["id"]
+def upload_file_to_drive(local_path, remote_name, folder_id):
+    try:
+        if not os.path.exists(local_path):
+            raise FileNotFoundError(f"❌ Local file does not exist: {local_path}")
+
+        # Delete existing file on Drive if present
+        existing = drive_service.files().list(
+            q=f"name='{remote_name}' and '{folder_id}' in parents and trashed = false",
+            fields="files(id)"
+        ).execute().get("files", [])
+
+        if existing:
+            drive_service.files().delete(fileId=existing[0]["id"]).execute()
+            print(f"🗑️ Deleted existing '{remote_name}' in Drive folder {folder_id}")
+
+        file_metadata = {"name": remote_name, "parents": [folder_id]}
+        media = MediaFileUpload(local_path, resumable=True)
+        file = drive_service.files().create(
+            body=file_metadata,
+            media_body=media,
+            fields="id"
+        ).execute()
+
+        print(f"✅ Uploaded '{remote_name}' to Google Drive folder ID: {folder_id}")
+        return file["id"]
+
+    except Exception as e:
+        print(f"❌ Error during upload: {e}")
+        return None
+
+
+def download_file_from_drive(remote_name, folder_id, local_path):
+    results = drive_service.files().list(q=f"name='{remote_name}' and '{folder_id}' in parents", fields="files(id)").execute()
+    files = results.get("files", [])
+    if not files:
+        return False
+    file_id = files[0]["id"]
+    request = drive_service.files().get_media(fileId=file_id)
+    fh = io.FileIO(local_path, 'wb')
+    downloader = MediaIoBaseDownload(fh, request)
+    done = False
+    while not done:
+        _, done = downloader.next_chunk()
+    return True
+def download_drive_file_content(file_id):
+    request = drive_service.files().get_media(fileId=file_id)
+    fh = io.BytesIO()
+    downloader = MediaIoBaseDownload(fh, request)
+    done = False
+    while not done:
+        _, done = downloader.next_chunk()
+    fh.seek(0)
+    return fh.read().decode("utf-8")

core/model.py

@@ -0,0 +1,1414 @@
+import re, os, json
+import pycountry, faiss
+from docx import Document
+import numpy as np
+from collections import defaultdict
+import ast # For literal_eval
+import math # For ceiling function
+import core.data_preprocess
+import core.mtdna_classifier
+# --- IMPORTANT: UNCOMMENT AND CONFIGURE YOUR REAL API KEY ---
+import google.generativeai as genai
+
+#genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
+genai.configure(api_key=os.getenv("GOOGLE_API_KEY_BACKUP"))
+
+import nltk
+from nltk.corpus import stopwords
+try:
+    nltk.data.find('corpora/stopwords')
+except LookupError:
+    nltk.download('stopwords')
+nltk.download('punkt_tab')    
+# # --- Define Pricing Constants (for Gemini 1.5 Flash & text-embedding-004) ---
+# # Prices are per 1,000 tokens
+# PRICE_PER_1K_INPUT_LLM = 0.000075  # $0.075 per 1M tokens
+# PRICE_PER_1K_OUTPUT_LLM = 0.0003   # $0.30 per 1M tokens
+# PRICE_PER_1K_EMBEDDING_INPUT = 0.000025 # $0.025 per 1M tokens
+
+# Gemini 2.5 Flash-Lite pricing per 1,000 tokens
+PRICE_PER_1K_INPUT_LLM = 0.00010      # $0.10 per 1M input tokens
+PRICE_PER_1K_OUTPUT_LLM = 0.00040     # $0.40 per 1M output tokens
+
+# Embedding-001 pricing per 1,000 input tokens
+PRICE_PER_1K_EMBEDDING_INPUT = 0.00015  # $0.15 per 1M input tokens
+# --- API Functions (REAL API FUNCTIONS) ---
+
+# def get_embedding(text, task_type="RETRIEVAL_DOCUMENT"):
+#     """Generates an embedding for the given text using a Google embedding model."""
+#     try:
+#         result = genai.embed_content(
+#             model="models/text-embedding-004", # Specify the embedding model
+#             content=text,
+#             task_type=task_type
+#         )
+#         return np.array(result['embedding']).astype('float32')
+#     except Exception as e:
+#         print(f"Error getting embedding: {e}")
+#         return np.zeros(768, dtype='float32')
+def get_embedding(text, task_type="RETRIEVAL_DOCUMENT"):
+    """Safe Gemini 1.5 embedding call with fallback."""
+    import numpy as np
+    try:
+        if not text or len(text.strip()) == 0:
+            raise ValueError("Empty text cannot be embedded.")
+        result = genai.embed_content(
+            model="models/text-embedding-004",
+            content=text,
+            task_type=task_type
+        )
+        return np.array(result['embedding'], dtype='float32')
+    except Exception as e:
+        print(f"❌ Embedding error: {e}")
+        return np.zeros(768, dtype='float32')
+
+
+def call_llm_api(prompt, model_name="gemini-2.5-flash-lite"):#'gemini-1.5-flash-latest'):
+    """Calls a Google Gemini LLM with the given prompt."""
+    try:
+        model = genai.GenerativeModel(model_name)
+        response = model.generate_content(prompt)
+        return response.text, model # Return model instance for token counting
+    except Exception as e:
+        print(f"Error calling LLM: {e}")
+        return "Error: Could not get response from LLM API.", None
+
+
+# --- Core Document Processing Functions (All previously provided and fixed) ---
+
+def read_docx_text(path):
+    """
+    Reads text and extracts potential table-like strings from a .docx document.
+    Separates plain text from structured [ [ ] ] list-like tables.
+    Also attempts to extract a document title.
+    """
+    doc = Document(path)
+    plain_text_paragraphs = []
+    table_strings = []
+    document_title = "Unknown Document Title" # Default
+
+    # Attempt to extract the document title from the first few paragraphs
+    title_paragraphs = [p.text.strip() for p in doc.paragraphs[:5] if p.text.strip()]
+    if title_paragraphs:
+        # A heuristic to find a title: often the first or second non-empty paragraph
+        # or a very long first paragraph if it's the title
+        if len(title_paragraphs[0]) > 50 and "Human Genetics" not in title_paragraphs[0]:
+            document_title = title_paragraphs[0]
+        elif len(title_paragraphs) > 1 and len(title_paragraphs[1]) > 50 and "Human Genetics" not in title_paragraphs[1]:
+            document_title = title_paragraphs[1]
+        elif any("Complete mitochondrial genomes" in p for p in title_paragraphs):
+            # Fallback to a known title phrase if present
+            document_title = "Complete mitochondrial genomes of Thai and Lao populations indicate an ancient origin of Austroasiatic groups and demic diffusion in the spread of Tai–Kadai languages"
+
+    current_table_lines = []
+    in_table_parsing_mode = False
+
+    for p in doc.paragraphs:
+        text = p.text.strip()
+        if not text:
+            continue
+
+        # Condition to start or continue table parsing
+        if text.startswith("## Table "): # Start of a new table section
+            if in_table_parsing_mode and current_table_lines:
+                table_strings.append("\n".join(current_table_lines))
+            current_table_lines = [text] # Include the "## Table X" line
+            in_table_parsing_mode = True
+        elif in_table_parsing_mode and (text.startswith("[") or text.startswith('"')):
+            # Continue collecting lines if we're in table mode and it looks like table data
+            # Table data often starts with '[' for lists, or '"' for quoted strings within lists.
+            current_table_lines.append(text)
+        else:
+            # If not in table mode, or if a line doesn't look like table data,
+            # then close the current table (if any) and add the line to plain text.
+            if in_table_parsing_mode and current_table_lines:
+                table_strings.append("\n".join(current_table_lines))
+                current_table_lines = []
+            in_table_parsing_mode = False
+            plain_text_paragraphs.append(text)
+
+    # After the loop, add any remaining table lines
+    if current_table_lines:
+        table_strings.append("\n".join(current_table_lines))
+
+    return "\n".join(plain_text_paragraphs), table_strings, document_title
+
+# --- Structured Data Extraction and RAG Functions ---
+
+def parse_literal_python_list(table_str):
+    list_match = re.search(r'(\[\s*\[\s*(?:.|\n)*?\s*\]\s*\])', table_str)
+    #print("Debug: list_match object (before if check):", list_match)
+    if not list_match:
+        if "table" in table_str.lower(): # then the table doest have the "]]" at the end
+            table_str += "]]"
+            list_match = re.search(r'(\[\s*\[\s*(?:.|\n)*?\s*\]\s*\])', table_str)
+    if list_match:
+        try:
+            matched_string = list_match.group(1)
+            #print("Debug: Matched string for literal_eval:", matched_string)
+            return ast.literal_eval(matched_string)
+        except (ValueError, SyntaxError) as e:
+            print(f"Error evaluating literal: {e}")
+            return []
+    return []
+
+
+_individual_code_parser = re.compile(r'([A-Z0-9]+?)(\d+)$', re.IGNORECASE)
+def _parse_individual_code_parts(code_str):
+    match = _individual_code_parser.search(code_str)
+    if match:
+        return match.group(1), match.group(2)
+    return None, None
+
+
+def parse_sample_id_to_population_code(plain_text_content):
+    sample_id_map = {}
+    contiguous_ranges_data = defaultdict(list)
+
+    #section_start_marker = "The sample identification of each population is as follows:"
+    section_start_marker = ["The sample identification of each population is as follows:","## table"]
+    
+    for s in section_start_marker:
+      relevant_text_search = re.search(
+          re.escape(s.lower()) + r"\s*(.*?)(?=\n##|\Z)",
+          plain_text_content.lower(),
+          re.DOTALL
+      )
+      if relevant_text_search: 
+        break
+      
+    if not relevant_text_search:
+        print("Warning: 'Sample ID Population Code' section start marker not found or block empty.")
+        return sample_id_map, contiguous_ranges_data
+
+    relevant_text_block = relevant_text_search.group(1).strip()
+
+    # print(f"\nDEBUG_PARSING: --- Start of relevant_text_block (first 500 chars) ---")
+    # print(relevant_text_block[:500])
+    # print(f"DEBUG_PARSING: --- End of relevant_text_block (last 500 chars) ---")
+    # print(relevant_text_block[-500:])
+    # print(f"DEBUG_PARSING: Relevant text block length: {len(relevant_text_block)}")
+
+    mapping_pattern = re.compile(
+    r'\b([A-Z0-9]+\d+)(?:-([A-Z0-9]+\d+))?\s+([A-Z0-9]+)\b', # Changed the last group
+    re.IGNORECASE)
+
+    range_expansion_count = 0
+    direct_id_count = 0
+    total_matches_found = 0
+    for match in mapping_pattern.finditer(relevant_text_block):
+        total_matches_found += 1
+        id1_full_str, id2_full_str_opt, pop_code = match.groups()
+
+        #print(f"  DEBUG_PARSING: Matched: '{match.group(0)}'")
+
+        pop_code_upper = pop_code.upper()
+
+        id1_prefix, id1_num_str = _parse_individual_code_parts(id1_full_str)
+        if id1_prefix is None:
+            #print(f"    DEBUG_PARSING: Failed to parse ID1: {id1_full_str}. Skipping this mapping.")
+            continue
+
+        if id2_full_str_opt:
+            id2_prefix_opt, id2_num_str_opt = _parse_individual_code_parts(id2_full_str_opt)
+            if id2_prefix_opt is None:
+                #print(f"    DEBUG_PARSING: Failed to parse ID2: {id2_full_str_opt}. Treating {id1_full_str} as single ID1.")
+                sample_id_map[f"{id1_prefix.upper()}{id1_num_str}"] = pop_code_upper
+                direct_id_count += 1
+                continue
+
+            #print(f"    DEBUG_PARSING: Comparing prefixes: '{id1_prefix.lower()}' vs '{id2_prefix_opt.lower()}'")
+            if id1_prefix.lower() == id2_prefix_opt.lower():
+                #print(f"    DEBUG_PARSING: ---> Prefixes MATCH for range expansion! Range: {id1_prefix}{id1_num_str}-{id2_prefix_opt}{id2_num_str_opt}")
+                try:
+                    start_num = int(id1_num_str)
+                    end_num = int(id2_num_str_opt)
+                    for num in range(start_num, end_num + 1):
+                        sample_id = f"{id1_prefix.upper()}{num}"
+                        sample_id_map[sample_id] = pop_code_upper
+                        range_expansion_count += 1
+                    contiguous_ranges_data[id1_prefix.upper()].append(
+                        (start_num, end_num, pop_code_upper)
+                    )
+                except ValueError:
+                    print(f"        DEBUG_PARSING: ValueError in range conversion for {id1_num_str}-{id2_num_str_opt}. Adding endpoints only.")
+                    sample_id_map[f"{id1_prefix.upper()}{id1_num_str}"] = pop_code_upper
+                    sample_id_map[f"{id2_prefix_opt.upper()}{id2_num_str_opt}"] = pop_code_upper
+                    direct_id_count += 2
+            else:
+                #print(f"    DEBUG_PARSING: Prefixes MISMATCH for range: '{id1_prefix}' vs '{id2_prefix_opt}'. Adding endpoints only.")
+                sample_id_map[f"{id1_prefix.upper()}{id1_num_str}"] = pop_code_upper
+                sample_id_map[f"{id2_prefix_opt.upper()}{id2_num_str_opt}"] = pop_code_upper
+                direct_id_count += 2
+        else:
+            sample_id_map[f"{id1_prefix.upper()}{id1_num_str}"] = pop_code_upper
+            direct_id_count += 1
+
+    # print(f"DEBUG_PARSING: Total matches found by regex: {total_matches_found}.")
+    # print(f"DEBUG_PARSING: Parsed sample IDs: {len(sample_id_map)} total entries.")
+    # print(f"DEBUG_PARSING:   (including {range_expansion_count} from range expansion and {direct_id_count} direct ID/endpoint entries).")
+    return sample_id_map, contiguous_ranges_data
+
+country_keywords_regional_overrides = {
+    "north thailand": "Thailand", "central thailand": "Thailand",
+    "northeast thailand": "Thailand", "east myanmar": "Myanmar", "west thailand": "Thailand",
+    "central india": "India", "east india": "India", "northeast india": "India",
+    "south sibera": "Russia", "siberia": "Russia", "yunnan": "China", #"tibet": "China",
+    "sumatra": "Indonesia", "borneo": "Indonesia",
+    "northern mindanao": "Philippines", "west malaysia": "Malaysia",
+    "mongolia": "China",
+    "beijing": "China",
+    "north laos": "Laos", "central laos": "Laos",
+    "east myanmar": "Myanmar", "west myanmar": "Myanmar"}
+
+# Updated get_country_from_text function
+def get_country_from_text(text):
+    text_lower = text.lower()
+
+    # 1. Use pycountry for official country names and common aliases
+    for country in pycountry.countries:
+        # Check full name match first
+        if text_lower == country.name.lower():
+            return country.name
+        
+        # Safely check for common_name
+        if hasattr(country, 'common_name') and text_lower == country.common_name.lower():
+            return country.common_name
+            
+        # Safely check for official_name
+        if hasattr(country, 'official_name') and text_lower == country.official_name.lower():
+            return country.official_name
+
+        # Check if country name is part of the text (e.g., 'Thailand' in 'Thailand border')
+        if country.name.lower() in text_lower:
+            return country.name
+            
+        # Safely check if common_name is part of the text
+        if hasattr(country, 'common_name') and country.common_name.lower() in text_lower:
+            return country.common_name
+    # 2. Prioritize specific regional overrides
+    for keyword, country in country_keywords_regional_overrides.items():
+        if keyword in text_lower:
+            return country
+    # 3. Check for broader regions that you want to map to "unknown" or a specific country
+    if "north asia" in text_lower or "southeast asia" in text_lower or "east asia" in text_lower:
+        return "unknown"
+
+    return "unknown"
+
+# Get the list of English stop words from NLTK
+non_meaningful_pop_names = set(stopwords.words('english'))
+
+def parse_population_code_to_country(plain_text_content, table_strings):
+    pop_code_country_map = {}
+    pop_code_ethnicity_map = {} # NEW: To store ethnicity for structured lookup
+    pop_code_specific_loc_map = {} # NEW: To store specific location for structured lookup
+
+    # Regex for parsing population info in structured lists and general text
+    # This pattern captures: (Pop Name/Ethnicity) (Pop Code) (Region/Specific Location) (Country) (Linguistic Family)
+    # The 'Pop Name/Ethnicity' (Group 1) is often the ethnicity
+    pop_info_pattern = re.compile(
+          r'([A-Za-z\s]+?)\s+([A-Z]+\d*)\s+'      # Pop Name (Group 1), Pop Code (Group 2) - Changed \d+ to \d* for codes like 'SH'
+          r'([A-Za-z\s\(\)\-,\/]+?)\s+'          # Region/Specific Location (Group 3)
+          r'(North+|South+|West+|East+|Thailand|Laos|Cambodia|Myanmar|Philippines|Indonesia|Malaysia|China|India|Taiwan|Vietnam|Russia|Nepal|Japan|South Korea)\b' # Country (Group 4)
+          r'(?:.*?([A-Za-z\s\-]+))?\s*'          # Optional Linguistic Family (Group 5), made optional with ?, followed by optional space
+          r'(\d+(?:\s+\d+\.?\d*)*)?', # Match all the numbers (Group 6) - made optional
+          re.IGNORECASE
+      )
+    for table_str in table_strings:
+        table_data = parse_literal_python_list(table_str)
+        if table_data:
+            is_list_of_lists = bool(table_data) and isinstance(table_data[0], list)
+            if is_list_of_lists:
+                for row_idx, row in enumerate(table_data):
+                    row_text = " ".join(map(str, row))
+                    match = pop_info_pattern.search(row_text)
+                    if match:
+                        pop_name = match.group(1).strip()
+                        pop_code = match.group(2).upper()
+                        specific_loc_text = match.group(3).strip()
+                        country_text = match.group(4).strip()
+                        linguistic_family = match.group(5).strip() if match.group(5) else 'unknown'
+
+                        final_country = get_country_from_text(country_text)
+                        if final_country == 'unknown': # Try specific loc text for country if direct country is not found
+                            final_country = get_country_from_text(specific_loc_text)
+
+                        if pop_code:
+                            pop_code_country_map[pop_code] = final_country
+
+                            # Populate ethnicity map (often Pop Name is ethnicity)
+                            pop_code_ethnicity_map[pop_code] = pop_name
+
+                            # Populate specific location map
+                            pop_code_specific_loc_map[pop_code] = specific_loc_text # Store as is from text
+            else:
+                row_text = " ".join(map(str, table_data))   
+                match = pop_info_pattern.search(row_text)
+                if match:
+                    pop_name = match.group(1).strip()
+                    pop_code = match.group(2).upper()
+                    specific_loc_text = match.group(3).strip()
+                    country_text = match.group(4).strip()
+                    linguistic_family = match.group(5).strip() if match.group(5) else 'unknown'
+
+                    final_country = get_country_from_text(country_text)
+                    if final_country == 'unknown': # Try specific loc text for country if direct country is not found
+                        final_country = get_country_from_text(specific_loc_text)
+
+                    if pop_code:
+                        pop_code_country_map[pop_code] = final_country
+
+                        # Populate ethnicity map (often Pop Name is ethnicity)
+                        pop_code_ethnicity_map[pop_code] = pop_name
+
+                        # Populate specific location map
+                        pop_code_specific_loc_map[pop_code] = specific_loc_text # Store as is from text
+
+                        # # Special case refinements for ethnicity/location if more specific rules are known from document:
+                        # if pop_name.lower() == "khon mueang": # and specific conditions if needed
+                        #     pop_code_ethnicity_map[pop_code] = "Khon Mueang"
+                        #     # If Khon Mueang has a specific city/district, add here
+                        #     # e.g., if 'Chiang Mai' is directly linked to KM1 in a specific table
+                        #     # pop_code_specific_loc_map[pop_code] = "Chiang Mai"
+                        # elif pop_name.lower() == "lawa":
+                        #      pop_code_ethnicity_map[pop_code] = "Lawa"
+                        # # Add similar specific rules for other populations (e.g., Mon for MO1, MO2, MO3)
+                        # elif pop_name.lower() == "mon":
+                        #     pop_code_ethnicity_map[pop_code] = "Mon"
+                        #     # For MO2: "West Thailand (Thailand Myanmar border)" -> no city
+                        #     # For MO3: "East Myanmar (Thailand Myanmar border)" -> no city
+                        #     # If the doc gives "Bangkok" for MO4, add it here for MO4's actual specific_location.
+                        # # etc.
+
+    # Fallback to parsing general plain text content (sentences)
+    sentences = data_preprocess.extract_sentences(plain_text_content)
+    for s in sentences: # Still focusing on just this one sentence
+      # Use re.finditer to get all matches
+      matches = pop_info_pattern.finditer(s)
+      pop_name, pop_code, specific_loc_text, country_text = "unknown", "unknown", "unknown", "unknown"
+      for match in matches:
+          if match.group(1):
+            pop_name = match.group(1).strip()
+          if match.group(2):  
+            pop_code = match.group(2).upper()
+          if match.group(3):  
+            specific_loc_text = match.group(3).strip()
+          if match.group(4):  
+            country_text = match.group(4).strip()
+          # linguistic_family = match.group(5).strip() if match.group(5) else 'unknown' # Already captured by pop_info_pattern
+
+          final_country = get_country_from_text(country_text)
+          if final_country == 'unknown':
+              final_country = get_country_from_text(specific_loc_text)
+
+          if pop_code.lower() not in non_meaningful_pop_names:
+            if final_country.lower() not in non_meaningful_pop_names:
+              pop_code_country_map[pop_code] = final_country
+            if pop_name.lower() not in non_meaningful_pop_names:  
+              pop_code_ethnicity_map[pop_code] = pop_name # Default ethnicity from Pop Name
+            if specific_loc_text.lower() not in non_meaningful_pop_names:  
+              pop_code_specific_loc_map[pop_code] = specific_loc_text
+
+              # Specific rules for ethnicity/location in plain text:
+              if pop_name.lower() == "khon mueang":
+                  pop_code_ethnicity_map[pop_code] = "Khon Mueang"
+              elif pop_name.lower() == "lawa":
+                  pop_code_ethnicity_map[pop_code] = "Lawa"
+              elif pop_name.lower() == "mon":
+                  pop_code_ethnicity_map[pop_code] = "Mon"
+              elif pop_name.lower() == "seak": # Added specific rule for Seak
+                  pop_code_ethnicity_map[pop_code] = "Seak"
+              elif pop_name.lower() == "nyaw": # Added specific rule for Nyaw
+                  pop_code_ethnicity_map[pop_code] = "Nyaw"
+              elif pop_name.lower() == "nyahkur": # Added specific rule for Nyahkur
+                  pop_code_ethnicity_map[pop_code] = "Nyahkur"
+              elif pop_name.lower() == "suay": # Added specific rule for Suay
+                  pop_code_ethnicity_map[pop_code] = "Suay"
+              elif pop_name.lower() == "soa": # Added specific rule for Soa
+                  pop_code_ethnicity_map[pop_code] = "Soa"
+              elif pop_name.lower() == "bru": # Added specific rule for Bru
+                  pop_code_ethnicity_map[pop_code] = "Bru"
+              elif pop_name.lower() == "khamu": # Added specific rule for Khamu
+                  pop_code_ethnicity_map[pop_code] = "Khamu"
+
+    return pop_code_country_map, pop_code_ethnicity_map, pop_code_specific_loc_map
+
+def general_parse_population_code_to_country(plain_text_content, table_strings):
+    pop_code_country_map = {}
+    pop_code_ethnicity_map = {}
+    pop_code_specific_loc_map = {}
+    sample_id_to_pop_code = {}
+
+    for table_str in table_strings:
+        table_data = parse_literal_python_list(table_str)
+        if not table_data or not isinstance(table_data[0], list):
+            continue
+
+        header_row = [col.lower() for col in table_data[0]]
+        header_map = {col: idx for idx, col in enumerate(header_row)}
+
+        # MJ17: Direct PopCode → Country
+        if 'id' in header_map and 'country' in header_map:
+            for row in table_strings[1:]:
+                row = parse_literal_python_list(row)[0]
+                if len(row) < len(header_row):
+                    continue
+                pop_code = str(row[header_map['id']]).strip()
+                country = str(row[header_map['country']]).strip()
+                province = row[header_map['province']].strip() if 'province' in header_map else 'unknown'
+                pop_group = row[header_map['population group / region']].strip() if 'population group / region' in header_map else 'unknown'
+                pop_code_country_map[pop_code] = country
+                pop_code_specific_loc_map[pop_code] = province
+                pop_code_ethnicity_map[pop_code] = pop_group
+
+        # A1YU101 or EBK/KSK: SampleID → PopCode
+        elif 'sample id' in header_map and 'population code' in header_map:
+            for row in table_strings[1:]:
+                row = parse_literal_python_list(row)[0]
+                if len(row) < 2:
+                    continue
+                sample_id = row[header_map['sample id']].strip().upper()
+                pop_code = row[header_map['population code']].strip().upper()
+                sample_id_to_pop_code[sample_id] = pop_code
+
+        # PopCode → Country (A1YU101/EBK mapping)
+        elif 'population code' in header_map and 'country' in header_map:
+            for row in table_strings[1:]:
+                row = parse_literal_python_list(row)[0]
+                if len(row) < 2:
+                    continue
+                pop_code = row[header_map['population code']].strip().upper()
+                country = row[header_map['country']].strip()
+                pop_code_country_map[pop_code] = country
+
+    return pop_code_country_map, pop_code_ethnicity_map, pop_code_specific_loc_map, sample_id_to_pop_code
+
+def chunk_text(text, chunk_size=500, overlap=50):
+    """Splits text into chunks (by words) with overlap."""
+    chunks = []
+    words = text.split()
+    num_words = len(words)
+
+    start = 0
+    while start < num_words:
+        end = min(start + chunk_size, num_words)
+        chunk = " ".join(words[start:end])
+        chunks.append(chunk)
+
+        if end == num_words:
+            break
+        start += chunk_size - overlap # Move start by (chunk_size - overlap)
+    return chunks
+
+def build_vector_index_and_data(doc_path, index_path="faiss_index.bin", chunks_path="document_chunks.json", structured_path="structured_lookup.json"):
+    """
+    Reads document, builds structured lookup, chunks remaining text, embeds chunks,
+    and builds/saves a FAISS index.
+    """
+    print("Step 1: Reading document and extracting structured data...")
+    # plain_text_content, table_strings, document_title = read_docx_text(doc_path) # Get document_title here
+
+    # sample_id_map, contiguous_ranges_data = parse_sample_id_to_population_code(plain_text_content)
+    # pop_code_to_country, pop_code_to_ethnicity, pop_code_to_specific_loc = parse_population_code_to_country(plain_text_content, table_strings)
+
+    # master_structured_lookup = {}
+    # master_structured_lookup['document_title'] = document_title # Store document title
+    # master_structured_lookup['sample_id_map'] = sample_id_map
+    # master_structured_lookup['contiguous_ranges'] = dict(contiguous_ranges_data)
+    # master_structured_lookup['pop_code_to_country'] = pop_code_to_country
+    # master_structured_lookup['pop_code_to_ethnicity'] = pop_code_to_ethnicity # NEW: Store pop_code to ethnicity map
+    # master_structured_lookup['pop_code_to_specific_loc'] = pop_code_to_specific_loc # NEW: Store pop_code to specific_loc map
+
+
+    # # Final consolidation: Use sample_id_map to derive full info for queries
+    # final_structured_entries = {}
+    # for sample_id, pop_code in master_structured_lookup['sample_id_map'].items():
+    #     country = master_structured_lookup['pop_code_to_country'].get(pop_code, 'unknown')
+    #     ethnicity = master_structured_lookup['pop_code_to_ethnicity'].get(pop_code, 'unknown') # Retrieve ethnicity
+    #     specific_location = master_structured_lookup['pop_code_to_specific_loc'].get(pop_code, 'unknown') # Retrieve specific location
+
+    #     final_structured_entries[sample_id] = {
+    #         'population_code': pop_code,
+    #         'country': country,
+    #         'type': 'modern',
+    #         'ethnicity': ethnicity, # Store ethnicity
+    #         'specific_location': specific_location # Store specific location
+    #     }
+    # master_structured_lookup['final_structured_entries'] = final_structured_entries
+    plain_text_content, table_strings, document_title = read_docx_text(doc_path)
+    pop_code_to_country, pop_code_to_ethnicity, pop_code_to_specific_loc, sample_id_map = general_parse_population_code_to_country(plain_text_content, table_strings)
+
+    final_structured_entries = {}
+    if sample_id_map:
+        for sample_id, pop_code in sample_id_map.items():
+            country = pop_code_to_country.get(pop_code, 'unknown')
+            ethnicity = pop_code_to_ethnicity.get(pop_code, 'unknown')
+            specific_loc = pop_code_to_specific_loc.get(pop_code, 'unknown')
+            final_structured_entries[sample_id] = {
+                'population_code': pop_code,
+                'country': country,
+                'type': 'modern',
+                'ethnicity': ethnicity,
+                'specific_location': specific_loc
+            }
+    else:
+        for pop_code in pop_code_to_country.keys():
+            country = pop_code_to_country.get(pop_code, 'unknown')
+            ethnicity = pop_code_to_ethnicity.get(pop_code, 'unknown')
+            specific_loc = pop_code_to_specific_loc.get(pop_code, 'unknown')
+            final_structured_entries[pop_code] = {
+                'population_code': pop_code,
+                'country': country,
+                'type': 'modern',
+                'ethnicity': ethnicity,
+                'specific_location': specific_loc
+            }
+    if not final_structured_entries:
+      # traditional way of A1YU101
+      sample_id_map, contiguous_ranges_data = parse_sample_id_to_population_code(plain_text_content)
+      pop_code_to_country, pop_code_to_ethnicity, pop_code_to_specific_loc = parse_population_code_to_country(plain_text_content, table_strings)
+      if sample_id_map:
+        for sample_id, pop_code in sample_id_map.items():
+            country = pop_code_to_country.get(pop_code, 'unknown')
+            ethnicity = pop_code_to_ethnicity.get(pop_code, 'unknown')
+            specific_loc = pop_code_to_specific_loc.get(pop_code, 'unknown')
+            final_structured_entries[sample_id] = {
+                'population_code': pop_code,
+                'country': country,
+                'type': 'modern',
+                'ethnicity': ethnicity,
+                'specific_location': specific_loc
+            }
+      else:
+          for pop_code in pop_code_to_country.keys():
+              country = pop_code_to_country.get(pop_code, 'unknown')
+              ethnicity = pop_code_to_ethnicity.get(pop_code, 'unknown')
+              specific_loc = pop_code_to_specific_loc.get(pop_code, 'unknown')
+              final_structured_entries[pop_code] = {
+                  'population_code': pop_code,
+                  'country': country,
+                  'type': 'modern',
+                  'ethnicity': ethnicity,
+                  'specific_location': specific_loc
+              }
+    
+    master_lookup = {
+        'document_title': document_title,
+        'pop_code_to_country': pop_code_to_country,
+        'pop_code_to_ethnicity': pop_code_to_ethnicity,
+        'pop_code_to_specific_loc': pop_code_to_specific_loc,
+        'sample_id_map': sample_id_map,
+        'final_structured_entries': final_structured_entries
+    }
+    print(f"Structured lookup built with {len(final_structured_entries)} entries in 'final_structured_entries'.")
+
+    with open(structured_path, 'w') as f:
+        json.dump(master_lookup, f, indent=4)
+    print(f"Structured lookup saved to {structured_path}.")
+
+    print("Step 2: Chunking document for RAG vector index...")
+    # replace the chunk here with the all_output from process_inputToken and fallback to this traditional chunk
+    clean_text, clean_table = "", ""
+    if plain_text_content:
+      clean_text = data_preprocess.normalize_for_overlap(plain_text_content)
+    if table_strings:
+      clean_table = data_preprocess.normalize_for_overlap(". ".join(table_strings))
+    all_clean_chunk = clean_text + clean_table
+    document_chunks = chunk_text(all_clean_chunk)
+    print(f"Document chunked into {len(document_chunks)} chunks.")
+    
+    print("Step 3: Generating embeddings for chunks (this might take time and cost API calls)...")
+
+    embedding_model_for_chunks = genai.GenerativeModel('models/text-embedding-004')
+
+    chunk_embeddings = []
+    for i, chunk in enumerate(document_chunks):
+        embedding = get_embedding(chunk, task_type="RETRIEVAL_DOCUMENT")
+        if embedding is not None and embedding.shape[0] > 0:
+            chunk_embeddings.append(embedding)
+        else:
+            print(f"Warning: Failed to get valid embedding for chunk {i}. Skipping.")
+            chunk_embeddings.append(np.zeros(768, dtype='float32'))
+
+    if not chunk_embeddings:
+        raise ValueError("No valid embeddings generated. Check get_embedding function and API.")
+
+    embedding_dimension = chunk_embeddings[0].shape[0]
+    index = faiss.IndexFlatL2(embedding_dimension)
+    index.add(np.array(chunk_embeddings))
+
+    faiss.write_index(index, index_path)
+    with open(chunks_path, "w") as f:
+        json.dump(document_chunks, f)
+
+    print(f"FAISS index built and saved to {index_path}.")
+    print(f"Document chunks saved to {chunks_path}.")
+    return master_lookup, index, document_chunks, all_clean_chunk
+
+
+def load_rag_assets(index_path="faiss_index.bin", chunks_path="document_chunks.json", structured_path="structured_lookup.json"):
+    """Loads pre-built RAG assets (FAISS index, chunks, structured lookup)."""
+    print("Loading RAG assets...")
+    master_structured_lookup = {}
+    if os.path.exists(structured_path):
+        with open(structured_path, 'r') as f:
+            master_structured_lookup = json.load(f)
+        print("Structured lookup loaded.")
+    else:
+        print("Structured lookup file not found. Rebuilding is likely needed.")
+
+    index = None
+    chunks = []
+    if os.path.exists(index_path) and os.path.exists(chunks_path):
+        try:
+            index = faiss.read_index(index_path)
+            with open(chunks_path, "r") as f:
+                chunks = json.load(f)
+            print("FAISS index and chunks loaded.")
+        except Exception as e:
+            print(f"Error loading FAISS index or chunks: {e}. Will rebuild.")
+            index = None
+            chunks = []
+    else:
+        print("FAISS index or chunks files not found.")
+
+    return master_structured_lookup, index, chunks
+# Helper function for query_document_info
+def exactInContext(text, keyword):
+# try keyword_prfix
+  # code_pattern = re.compile(r'([A-Z0-9]+?)(\d+)$', re.IGNORECASE)
+  # # Attempt to parse the keyword into its prefix and numerical part using re.search
+  # keyword_match = code_pattern.search(keyword)
+  # keyword_prefix = None
+  # keyword_num = None
+  # if keyword_match:
+  #     keyword_prefix = keyword_match.group(1).lower()
+  #     keyword_num = int(keyword_match.group(2))
+  text = text.lower()
+  idx = text.find(keyword.lower())
+  if idx == -1:
+    # if keyword_prefix:
+    #   idx = text.find(keyword_prefix)
+    # if idx == -1:
+    #   return False
+    return False
+  return True
+def chooseContextLLM(contexts, kw):
+  # if kw in context
+  for con in contexts:
+    context = contexts[con]
+    if context:
+      if exactInContext(context, kw):
+        return con, context    
+  #if cannot find anything related to kw in context, return all output
+  if contexts["all_output"]:
+    return "all_output", contexts["all_output"]
+  else:
+    # if all_output not exist
+    # look of chunk and still not exist return document chunk
+    if contexts["chunk"]: return "chunk", contexts["chunk"]
+    elif contexts["document_chunk"]:  return "document_chunk", contexts["document_chunk"]
+    else: return None, None  
+def clean_llm_output(llm_response_text, output_format_str):
+    results = []
+    lines = llm_response_text.strip().split('\n')
+    output_country, output_type, output_ethnicity, output_specific_location = [],[],[],[]
+    for line in lines:
+        extracted_country, extracted_type, extracted_ethnicity, extracted_specific_location = "unknown", "unknown", "unknown", "unknown"
+        line = line.strip()
+        if output_format_str == "ethnicity, specific_location/unknown": # Targeted RAG output
+            parsed_output = re.search(r'^\s*([^,]+?),\s*(.+?)\s*$', llm_response_text)
+            if parsed_output:
+                extracted_ethnicity = parsed_output.group(1).strip()
+                extracted_specific_location = parsed_output.group(2).strip()
+            else:
+                print("  DEBUG: LLM did not follow expected 2-field format for targeted RAG. Defaulting to unknown for ethnicity/specific_location.")
+                extracted_ethnicity = 'unknown'
+                extracted_specific_location = 'unknown'
+        elif output_format_str == "modern/ancient/unknown, ethnicity, specific_location/unknown":
+          parsed_output = re.search(r'^\s*([^,]+?),\s*([^,]+?),\s*(.+?)\s*$', llm_response_text)
+          if parsed_output:
+              extracted_type = parsed_output.group(1).strip()
+              extracted_ethnicity = parsed_output.group(2).strip()
+              extracted_specific_location = parsed_output.group(3).strip()
+          else:
+              # Fallback: check if only 2 fields
+              parsed_output_2_fields = re.search(r'^\s*([^,]+?),\s*([^,]+?)\s*$', llm_response_text)
+              if parsed_output_2_fields:
+                  extracted_type = parsed_output_2_fields.group(1).strip()
+                  extracted_ethnicity = parsed_output_2_fields.group(2).strip()
+                  extracted_specific_location = 'unknown'
+              else:
+                  # even simpler fallback: 1 field only
+                  parsed_output_1_field = re.search(r'^\s*([^,]+?)\s*$', llm_response_text)
+                  if parsed_output_1_field:
+                      extracted_type = parsed_output_1_field.group(1).strip()
+                      extracted_ethnicity = 'unknown'
+                      extracted_specific_location = 'unknown'
+                  else:
+                      print("  DEBUG: LLM did not follow any expected simplified format. Attempting verbose parsing fallback.")
+                      type_match_fallback = re.search(r'Type:\s*([A-Za-z\s-]+)', llm_response_text)
+                      extracted_type = type_match_fallback.group(1).strip() if type_match_fallback else 'unknown'
+                      extracted_ethnicity = 'unknown'
+                      extracted_specific_location = 'unknown'
+        else:
+          parsed_output = re.search(r'^\s*([^,]+?),\s*([^,]+?),\s*([^,]+?),\s*(.+?)\s*$', line)
+          if parsed_output:
+              extracted_country = parsed_output.group(1).strip()
+              extracted_type = parsed_output.group(2).strip()
+              extracted_ethnicity = parsed_output.group(3).strip()
+              extracted_specific_location = parsed_output.group(4).strip()
+          else:
+              print(f"  DEBUG: Line did not follow expected 4-field format: {line}")
+              parsed_output_2_fields = re.search(r'^\s*([^,]+?),\s*([^,]+?)\s*$', line)
+              if parsed_output_2_fields:
+                  extracted_country = parsed_output_2_fields.group(1).strip()
+                  extracted_type = parsed_output_2_fields.group(2).strip()
+                  extracted_ethnicity = 'unknown'
+                  extracted_specific_location = 'unknown'
+              else:
+                  print(f"  DEBUG: Fallback to verbose-style parsing: {line}")
+                  country_match_fallback = re.search(r'Country:\s*([A-Za-z\s-]+)', line)
+                  type_match_fallback = re.search(r'Type:\s*([A-Za-z\s-]+)', line)
+                  extracted_country = country_match_fallback.group(1).strip() if country_match_fallback else 'unknown'
+                  extracted_type = type_match_fallback.group(1).strip() if type_match_fallback else 'unknown'
+                  extracted_ethnicity = 'unknown'
+                  extracted_specific_location = 'unknown'
+
+        results.append({
+            "country": extracted_country,
+            "type": extracted_type,
+            "ethnicity": extracted_ethnicity,
+            "specific_location": extracted_specific_location
+            #"country_explain":extracted_country_explain,
+            #"type_explain": extracted_type_explain
+        })
+    # if more than 2 results
+    if output_format_str == "ethnicity, specific_location/unknown":
+      for result in results:
+        if result["ethnicity"] not in output_ethnicity:
+          output_ethnicity.append(result["ethnicity"])
+        if result["specific_location"] not in output_specific_location:  
+          output_specific_location.append(result["specific_location"])
+      return " or ".join(output_ethnicity), " or ".join(output_specific_location)     
+    elif output_format_str == "modern/ancient/unknown, ethnicity, specific_location/unknown":
+      for result in results:
+        if result["type"] not in output_type:
+          output_type.append(result["type"])
+        if result["ethnicity"] not in output_ethnicity:
+          output_ethnicity.append(result["ethnicity"])
+        if result["specific_location"] not in output_specific_location:  
+          output_specific_location.append(result["specific_location"])
+
+      return " or ".join(output_type)," or ".join(output_ethnicity), " or ".join(output_specific_location)    
+    else:
+      for result in results:
+        if result["country"] not in output_country:
+          output_country.append(result["country"])
+        if result["type"] not in output_type:
+          output_type.append(result["type"])
+        if result["ethnicity"] not in output_ethnicity:
+          output_ethnicity.append(result["ethnicity"])
+        if result["specific_location"] not in output_specific_location:  
+          output_specific_location.append(result["specific_location"])
+      return " or ".join(output_country)," or ".join(output_type)," or ".join(output_ethnicity), " or ".join(output_specific_location)           
+
+# def parse_multi_sample_llm_output(raw_response: str, output_format_str):
+#     """
+#     Parse LLM output with possibly multiple metadata lines + shared explanations.
+#     """
+#     lines = [line.strip() for line in raw_response.strip().splitlines() if line.strip()]
+#     metadata_list = []
+#     explanation_lines = []
+#     if output_format_str == "country_name, modern/ancient/unknown":
+#         parts = [x.strip() for x in lines[0].split(",")]
+#         if len(parts)==2:
+#           metadata_list.append({
+#               "country": parts[0],
+#               "sample_type": parts[1]#,
+#               #"ethnicity": parts[2],
+#               #"location": parts[3]
+#           })
+#         if 1<len(lines):
+#           line = lines[1]
+#           if "\n" in line:  line = line.split("\n")
+#           if ". " in line: line = line.split(". ")
+#           if isinstance(line,str): line = [line]
+#           explanation_lines += line
+#     elif output_format_str == "modern/ancient/unknown":
+#       metadata_list.append({
+#           "country": "unknown",
+#           "sample_type": lines[0]#,
+#           #"ethnicity": parts[2],
+#           #"location": parts[3]
+#       })
+#       explanation_lines.append(lines[1])
+
+#     # Assign explanations (optional) to each sample — same explanation reused
+#     for md in metadata_list:
+#         md["country_explanation"] = None
+#         md["sample_type_explanation"] = None
+
+#         if md["country"].lower() != "unknown" and len(explanation_lines) >= 1:
+#             md["country_explanation"] = explanation_lines[0]
+
+#         if md["sample_type"].lower() != "unknown":
+#             if len(explanation_lines) >= 2:
+#                 md["sample_type_explanation"] = explanation_lines[1]
+#             elif len(explanation_lines) == 1 and md["country"].lower() == "unknown":
+#                 md["sample_type_explanation"] = explanation_lines[0]
+#             elif len(explanation_lines) == 1:
+#                 md["sample_type_explanation"] = explanation_lines[0]
+#     return metadata_list
+
+def parse_multi_sample_llm_output(raw_response: str, output_format_str):
+    """
+    Parse LLM output with possibly multiple metadata lines + shared explanations.
+    """
+    metadata_list = {}
+    explanation_lines = []
+    output_answers = raw_response.split("\n")[0].split(", ")
+    explanation_lines =  [x for x in raw_response.split("\n")[1:] if x.strip()]
+    print("raw explanation line which split by new line: ", explanation_lines)
+    if len(explanation_lines) == 1:
+        if len(explanation_lines[0].split(". ")) > len(explanation_lines):
+          explanation_lines =  [x for x in explanation_lines[0].split(". ") if x.strip()]
+          print("explain line split by dot: ", explanation_lines)  
+    output_formats = output_format_str.split(", ")
+    explain = ""
+    # assign output format to its output answer and explanation
+    if output_format_str:
+      outputs = output_format_str.split(", ")
+      for o in range(len(outputs)):
+        output = outputs[o]
+        metadata_list[output] = {"answer":"",
+                                  output+"_explanation":""}   
+        # assign output answers
+        if o < len(output_answers):
+          # check if output_format unexpectedly in the answer such as:  
+          #country_name: Europe, modern/ancient: modern
+          try: 
+            if ": " in output_answers[o]:
+              output_answers[o] = output_answers[o].split(": ")[1]
+          except:
+            pass    
+          # Europe, modern  
+          metadata_list[output]["answer"] = output_answers[o]
+          if "unknown" in metadata_list[output]["answer"].lower():
+            metadata_list[output]["answer"] = "unknown"
+        else:
+          metadata_list[output]["answer"] = "unknown"
+        # assign explanations
+        if metadata_list[output]["answer"] != "unknown":
+          if explanation_lines:
+            explain = explanation_lines.pop(0)
+          metadata_list[output][output+"_explanation"] = explain
+        else:  
+          metadata_list[output][output+"_explanation"] = "unknown"  
+    return metadata_list
+
+def merge_metadata_outputs(metadata_list):
+    """
+    Merge a list of metadata dicts into one, combining differing values with 'or'.
+    Assumes all dicts have the same keys.
+    """
+    if not metadata_list:
+        return {}
+
+    merged = {}
+    keys = metadata_list[0].keys()
+
+    for key in keys:
+        values = [md[key] for md in metadata_list if key in md]
+        unique_values = list(dict.fromkeys(values))  # preserve order, remove dupes
+        if "unknown" in unique_values:
+          unique_values.pop(unique_values.index("unknown"))
+        if len(unique_values) == 1:
+            merged[key] = unique_values[0]
+        else:
+            merged[key] = " or ".join(unique_values)
+
+    return merged
+
+
+def query_document_info(query_word, alternative_query_word, metadata, master_structured_lookup, faiss_index, document_chunks, llm_api_function, chunk=None, all_output=None, model_ai=None):
+    """
+    Queries the document using a hybrid approach:
+    1. Local structured lookup (fast, cheap, accurate for known patterns).
+    2. RAG with semantic search and LLM (general, flexible, cost-optimized).
+    """
+    if model_ai:
+      if model_ai == "gemini-1.5-flash-latest":
+        genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
+        PRICE_PER_1K_INPUT_LLM = 0.000075  # $0.075 per 1M tokens
+        PRICE_PER_1K_OUTPUT_LLM = 0.0003   # $0.30 per 1M tokens
+        PRICE_PER_1K_EMBEDDING_INPUT = 0.000025 # $0.025 per 1M tokens
+        global_llm_model_for_counting_tokens = genai.GenerativeModel("gemini-1.5-flash-latest")#('gemini-1.5-flash-latest')
+    else:
+      genai.configure(api_key=os.getenv("GOOGLE_API_KEY_BACKUP"))    
+      # Gemini 2.5 Flash-Lite pricing per 1,000 tokens
+      PRICE_PER_1K_INPUT_LLM = 0.00010      # $0.10 per 1M input tokens
+      PRICE_PER_1K_OUTPUT_LLM = 0.00040     # $0.40 per 1M output tokens
+
+      # Embedding-001 pricing per 1,000 input tokens
+      PRICE_PER_1K_EMBEDDING_INPUT = 0.00015  # $0.15 per 1M input tokens
+      global_llm_model_for_counting_tokens = genai.GenerativeModel("gemini-2.5-flash-lite")#('gemini-1.5-flash-latest')
+    
+    if metadata:
+      extracted_country, extracted_specific_location, extracted_ethnicity, extracted_type = metadata["country"], metadata["specific_location"], metadata["ethnicity"], metadata["sample_type"]
+      extracted_col_date, extracted_iso, extracted_title, extracted_features = metadata["collection_date"], metadata["isolate"], metadata["title"], metadata["all_features"]
+    else:
+      extracted_country, extracted_specific_location, extracted_ethnicity, extracted_type = "unknown", "unknown", "unknown", "unknown"
+      extracted_col_date, extracted_iso, extracted_title = "unknown", "unknown", "unknown"
+    # --- NEW: Pre-process alternative_query_word to remove '.X' suffix if present ---
+    if alternative_query_word:
+        alternative_query_word_cleaned = alternative_query_word.split('.')[0]
+    else:
+        alternative_query_word_cleaned = alternative_query_word
+    country_explanation, sample_type_explanation = None, None
+
+    # Use the consolidated final_structured_entries for direct lookup
+    final_structured_entries = master_structured_lookup.get('final_structured_entries', {})
+    document_title = master_structured_lookup.get('document_title', 'Unknown Document Title') # Retrieve document title
+
+    # Default values for all extracted fields. These will be updated.
+    method_used = 'unknown' # Will be updated based on the method that yields a result
+    population_code_from_sl = 'unknown' # To pass to RAG prompt if available
+    total_query_cost = 0
+    # Attempt 1: Try primary query_word (e.g., isolate name) with structured lookup
+    try: 
+        print("try attempt 1 in model query")
+        structured_info = final_structured_entries.get(query_word.upper())
+        if structured_info:
+            if extracted_country == 'unknown':
+              extracted_country = structured_info['country']
+            if extracted_type == 'unknown':
+              extracted_type = structured_info['type']
+              
+            # if extracted_ethnicity == 'unknown':
+            #   extracted_ethnicity = structured_info.get('ethnicity', 'unknown') # Get ethnicity from structured lookup
+            # if extracted_specific_location == 'unknown':
+            #   extracted_specific_location = structured_info.get('specific_location', 'unknown') # Get specific_location from structured lookup
+            population_code_from_sl = structured_info['population_code']
+            method_used = "structured_lookup_direct"
+            print(f"'{query_word}' found in structured lookup (direct match).")
+    except:
+        print("pass attempt 1 in model query")
+        pass 
+    # Attempt 2: Try primary query_word with heuristic range lookup if direct fails (only if not already resolved)
+    try:
+        print("try attempt 2 in model query")
+        if method_used == 'unknown':
+            query_prefix, query_num_str = _parse_individual_code_parts(query_word)
+            if query_prefix is not None and query_num_str is not None:
+                try: query_num = int(query_num_str)
+                except ValueError: query_num = None
+                if query_num is not None:
+                    query_prefix_upper = query_prefix.upper()
+                    contiguous_ranges = master_structured_lookup.get('contiguous_ranges', defaultdict(list))
+                    pop_code_to_country = master_structured_lookup.get('pop_code_to_country', {})
+                    pop_code_to_ethnicity = master_structured_lookup.get('pop_code_to_ethnicity', {})
+                    pop_code_to_specific_loc = master_structured_lookup.get('pop_code_to_specific_loc', {})
+    
+                    if query_prefix_upper in contiguous_ranges:
+                        for start_num, end_num, pop_code_for_range in contiguous_ranges[query_prefix_upper]:
+                            if start_num <= query_num <= end_num:
+                                country_from_heuristic = pop_code_to_country.get(pop_code_for_range, 'unknown')
+                                if country_from_heuristic != 'unknown':
+                                    if extracted_country == 'unknown':
+                                      extracted_country = country_from_heuristic
+                                    if extracted_type == 'unknown':
+                                      extracted_type = 'modern'
+                                    # if extracted_ethnicity == 'unknown':
+                                    #   extracted_ethnicity = pop_code_to_ethnicity.get(pop_code_for_range, 'unknown')
+                                    # if extracted_specific_location == 'unknown':
+                                    #   extracted_specific_location = pop_code_to_specific_loc.get(pop_code_for_range, 'unknown')
+                                    population_code_from_sl = pop_code_for_range
+                                    method_used = "structured_lookup_heuristic_range_match"
+                                    print(f"'{query_word}' not direct. Heuristic: Falls within range {query_prefix_upper}{start_num}-{query_prefix_upper}{end_num}.")
+                                    break
+                                else:
+                                    print(f"'{query_word}' heuristic match found, but country unknown. Will fall to RAG below.")
+    except:
+        print("pass attempt 2 in model query")                              
+        pass
+    # Attempt 3: If primary query_word failed all structured lookups, try alternative_query_word (cleaned)
+    try:
+        print("try attempt 3 in model query")
+        if method_used == 'unknown' and alternative_query_word_cleaned and alternative_query_word_cleaned != query_word:
+            print(f"'{query_word}' not found in structured (or heuristic). Trying alternative '{alternative_query_word_cleaned}'.")
+    
+            # Try direct lookup for alternative word
+            structured_info_alt = final_structured_entries.get(alternative_query_word_cleaned.upper())
+            if structured_info_alt:
+              if extracted_country == 'unknown':
+                extracted_country = structured_info_alt['country']
+              if extracted_type == 'unknown':
+                extracted_type = structured_info_alt['type']
+              # if extracted_ethnicity == 'unknown':
+              #   extracted_ethnicity = structured_info_alt.get('ethnicity', 'unknown')
+              # if extracted_specific_location == 'unknown':
+              #   extracted_specific_location = structured_info_alt.get('specific_location', 'unknown')
+              population_code_from_sl = structured_info_alt['population_code']
+              method_used = "structured_lookup_alt_direct"
+              print(f"Alternative '{alternative_query_word_cleaned}' found in structured lookup (direct match).")
+            else:
+                # Try heuristic lookup for alternative word
+                alt_prefix, alt_num_str = _parse_individual_code_parts(alternative_query_word_cleaned)
+                if alt_prefix is not None and alt_num_str is not None:
+                    try: alt_num = int(alt_num_str)
+                    except ValueError: alt_num = None
+                    if alt_num is not None:
+                        alt_prefix_upper = alt_prefix.upper()
+                        contiguous_ranges = master_structured_lookup.get('contiguous_ranges', defaultdict(list))
+                        pop_code_to_country = master_structured_lookup.get('pop_code_to_country', {})
+                        pop_code_to_ethnicity = master_structured_lookup.get('pop_code_to_ethnicity', {})
+                        pop_code_to_specific_loc = master_structured_lookup.get('pop_code_to_specific_loc', {})
+                        if alt_prefix_upper in contiguous_ranges:
+                            for start_num, end_num, pop_code_for_range in contiguous_ranges[alt_prefix_upper]:
+                                if start_num <= alt_num <= end_num:
+                                    country_from_heuristic_alt = pop_code_to_country.get(pop_code_for_range, 'unknown')
+                                    if country_from_heuristic_alt != 'unknown':
+                                      if extracted_country == 'unknown':
+                                        extracted_country = country_from_heuristic_alt
+                                      if extracted_type == 'unknown':
+                                        extracted_type = 'modern'
+                                      # if extracted_ethnicity == 'unknown':
+                                      #   extracted_ethnicity = pop_code_to_ethnicity.get(pop_code_for_range, 'unknown')
+                                      # if extracted_specific_location == 'unknown':
+                                      #   extracted_specific_location = pop_code_to_specific_loc.get(pop_code_for_range, 'unknown')
+                                      population_code_from_sl = pop_code_for_range
+                                      method_used = "structured_lookup_alt_heuristic_range_match"
+                                      break
+                                    else:
+                                        print(f"Alternative '{alternative_query_word_cleaned}' heuristic match found, but country unknown. Will fall to RAG below.")
+    except:
+        print("pass attempt 3 in model query")
+        pass
+    # use the context_for_llm to detect present_ancient before using llm model
+    # retrieved_chunks_text = []
+    # if document_chunks:
+    #   for idx in range(len(document_chunks)):
+    #           retrieved_chunks_text.append(document_chunks[idx])
+    # context_for_llm = ""
+    # all_context = "\n".join(retrieved_chunks_text) # 
+    # listOfcontexts = {"chunk": chunk, 
+    #         "all_output": all_output,
+    #         "document_chunk": all_context}
+    # label, context_for_llm = chooseContextLLM(listOfcontexts, query_word) 
+    # if not context_for_llm:
+    #   label, context_for_llm = chooseContextLLM(listOfcontexts, alternative_query_word_cleaned)
+    #   if not context_for_llm:
+    #     context_for_llm = "Collection_date: " + col_date +". Isolate: " + iso + ". Title: " + title + ". Features: " + extracted_features
+    # if context_for_llm:
+    #   extracted_type, explain = mtdna_classifier.detect_ancient_flag(context_for_llm)
+    #   extracted_type = extracted_type.lower()
+    #   sample_type_explanation = explain
+    # 5. Execute RAG if needed (either full RAG or targeted RAG for missing fields)
+
+    # Determine if a RAG call is necessary
+    # run_rag = (extracted_country == 'unknown' or extracted_type == 'unknown')# or \
+    #            #extracted_ethnicity == 'unknown' or extracted_specific_location == 'unknown')
+    run_rag = True
+    if run_rag:
+        print("try run rag")
+        # Determine the phrase for LLM query
+        rag_query_phrase = f"'{query_word}'"
+        if alternative_query_word_cleaned and alternative_query_word_cleaned != query_word:
+            rag_query_phrase += f" or its alternative word '{alternative_query_word_cleaned}'"
+
+        # Construct a more specific semantic query phrase for embedding if structured info is available
+        semantic_query_for_embedding = rag_query_phrase # Default
+        # if extracted_country != 'unknown': # If country is known from structured lookup (for targeted RAG)
+        #     if population_code_from_sl != 'unknown':
+        #         semantic_query_for_embedding = f"ethnicity and specific location for {query_word} population {population_code_from_sl} in {extracted_country}"
+        #     else: # If pop_code not found in structured, still use country hint
+        #         semantic_query_for_embedding = f"ethnicity and specific location for {query_word} in {extracted_country}"
+        # print(f"  DEBUG: Semantic query for embedding: '{semantic_query_for_embedding}'")
+
+
+        # Determine fields to ask LLM for and output format based on what's known/needed
+        prompt_instruction_prefix = ""
+        output_format_str = ""
+
+        # Determine if it's a full RAG or targeted RAG scenario based on what's already extracted
+        is_full_rag_scenario = True#(extracted_country == 'unknown')
+
+        if is_full_rag_scenario: # Full RAG scenario
+            output_format_str = "country_name, modern/ancient/unknown"#, ethnicity, specific_location/unknown"
+            method_used = "rag_llm"
+            print(f"Proceeding to FULL RAG for {rag_query_phrase}.")
+        # else: # Targeted RAG scenario (country/type already known, need ethnicity/specific_location)
+        #     if extracted_type == "unknown":
+        #         prompt_instruction_prefix = (
+        #             f"I already know the country is {extracted_country}. "
+        #             f"{f'The population code is {population_code_from_sl}. ' if population_code_from_sl != 'unknown' else ''}"
+        #         )
+        #         #output_format_str = "modern/ancient/unknown, ethnicity, specific_location/unknown"
+        #         output_format_str = "modern/ancient/unknown"
+        #     # else:
+        #     #     prompt_instruction_prefix = (
+        #     #         f"I already know the country is {extracted_country} and the sample type is {extracted_type}. "
+        #     #         f"{f'The population code is {population_code_from_sl}. ' if population_code_from_sl != 'unknown' else ''}"
+        #     #     )
+        #     #     output_format_str = "ethnicity, specific_location/unknown"
+
+        #     method_used = "hybrid_sl_rag"
+        #     print(f"Proceeding to TARGETED RAG for {rag_query_phrase}.")
+
+
+        # Calculate embedding cost for the primary query word
+        current_embedding_cost = 0
+        try:
+            query_embedding_vector = get_embedding(semantic_query_for_embedding, task_type="RETRIEVAL_QUERY")
+            query_embedding_tokens = global_llm_model_for_counting_tokens.count_tokens(semantic_query_for_embedding).total_tokens
+            current_embedding_cost += (query_embedding_tokens / 1000) * PRICE_PER_1K_EMBEDDING_INPUT
+            print(f"  DEBUG: Query embedding tokens (for '{semantic_query_for_embedding}'): {query_embedding_tokens}, cost: ${current_embedding_cost:.6f}")
+
+            if alternative_query_word_cleaned and alternative_query_word_cleaned != query_word:
+                alt_embedding_vector = get_embedding(alternative_query_word_cleaned, task_type="RETRIEVAL_QUERY")
+                alt_embedding_tokens = global_llm_model_for_counting_tokens.count_tokens(alternative_query_word_cleaned).total_tokens
+                current_embedding_cost += (alt_embedding_tokens / 1000) * PRICE_PER_1K_EMBEDDING_INPUT
+                print(f"  DEBUG: Alternative query ('{alternative_query_word_cleaned}') embedding tokens: {alt_embedding_tokens}, cost: ${current_embedding_cost:.6f}")
+
+        except Exception as e:
+            print(f"Error getting query embedding for RAG: {e}")
+            return extracted_country, extracted_type, "embedding_failed", extracted_ethnicity, extracted_specific_location, total_query_cost
+
+        if query_embedding_vector is None or query_embedding_vector.shape[0] == 0:
+            return extracted_country, extracted_type, "embedding_failed", extracted_ethnicity, extracted_specific_location, total_query_cost
+
+        D, I = faiss_index.search(np.array([query_embedding_vector]), 4)
+
+        retrieved_chunks_text = []
+        for idx in I[0]:
+            if 0 <= idx < len(document_chunks):
+                retrieved_chunks_text.append(document_chunks[idx])
+
+        context_for_llm = ""
+        
+        all_context = "\n".join(retrieved_chunks_text) # 
+        listOfcontexts = {"chunk": chunk, 
+            "all_output": all_output,
+            "document_chunk": all_context}
+        label, context_for_llm = chooseContextLLM(listOfcontexts, query_word) 
+        if not context_for_llm:
+          label, context_for_llm = chooseContextLLM(listOfcontexts, alternative_query_word_cleaned)
+          if not context_for_llm:
+            context_for_llm = "Collection_date: " + col_date +". Isolate: " + iso + ". Title: " + title + ". Features: " + extracted_features
+        #print("context for llm: ", label)    
+        # prompt_for_llm = (
+        #     f"{prompt_instruction_prefix}"
+        #     f"Given the following text snippets, analyze the entity/concept {rag_query_phrase} or the mitochondrial DNA sample in general if these specific identifiers are not explicitly found. "
+        #     f"Identify its primary associated country/geographic location. "
+        #     f"Also, determine if the genetic sample or individual mentioned is from a 'modern' (present-day living individual) "
+        #     f"or 'ancient' (e.g., prehistoric remains, archaeological sample) source. "
+        #     f"If the text does not mention whether the sample is ancient or modern, assume the sample is modern unless otherwise explicitly described as ancient or archaeological. "
+        #     f"Additionally, extract its ethnicity and a more specific location (city/district level) within the predicted country. "
+        #     f"If any information is not explicitly present in the provided text snippets, state 'unknown' for that specific piece of information. "
+        #     f"Provide only the country, sample type, ethnicity, and specific location, do not add extra explanations.\n\n"
+        #     f"Text Snippets:\n{context_for_llm}\n\n"
+        #     f"Output Format: {output_format_str}"
+        # )
+        if len(context_for_llm) > 1000*1000:
+          context_for_llm = context_for_llm[:900000]
+
+        # fix the prompt better:
+        # firstly clarify more by saying which type of organism, prioritize homo sapiens
+        features = metadata["all_features"]
+        organism = "general"
+        if features != "unknown":
+          if "organism" in features:
+            try:
+              organism = features.split("organism: ")[1].split("\n")[0]  
+            except:
+              organism = features.replace("\n","; ")  
+        explain_list = "country or sample type (modern/ancient)" #or ethnicity or specific location (province/city)"    
+        
+#         prompt_for_llm = (
+#     f"{prompt_instruction_prefix}"
+#     f"Given the following text snippets, analyze the entity/concept {rag_query_phrase} or the mitochondrial DNA sample in general if these specific identifiers are not explicitly found. "
+#     f"Identify its primary associated country/geographic location. "
+#     f"Also, determine if the genetic sample or individual mentioned is from a 'modern' (present-day living individual) "
+#     f"or 'ancient' (e.g., prehistoric remains, archaeological sample) source. "
+#     f"If the text does not mention whether the sample is ancient or modern, assume the sample is modern unless otherwise explicitly described as ancient or archaeological. "
+#     f"Provide only {output_format_str}. "
+#     f"If any information is not explicitly present in the provided text snippets, state 'unknown' for that specific piece of information. "
+#     f"If the country or sample type (modern/ancient) is not 'unknown', write 1 sentence after the output explaining how you inferred it from the text (one sentence for each)."
+#     f"\n\nText Snippets:\n{context_for_llm}\n\n"
+#     f"Output Format: {output_format_str}"
+# )
+        
+#         prompt_for_llm = (
+#     f"{prompt_instruction_prefix}"
+#     f"Given the following text snippets, analyze the entity/concept {rag_query_phrase} or the mitochondrial DNA sample in {organism} if these specific identifiers are not explicitly found. "
+#     f"Identify its primary associated country/geographic location. "
+#     f"Also, determine if the genetic sample or individual mentioned is from a 'modern' (present-day living individual) "
+#     f"or 'ancient' (e.g., prehistoric remains, archaeological sample) source. "
+#     f"If the text does not mention whether the sample is ancient or modern, assume the sample is modern unless otherwise explicitly described as ancient or archaeological. "
+#     f"Provide only {output_format_str}. "
+#     f"If any information is not explicitly present in the provided text snippets, state 'unknown' for that specific piece of information. "
+#     f"If the {explain_list} is not 'unknown', write 1 sentence after the output explaining how you inferred it from the text (one sentence for each)."
+#     f"\n\nText Snippets:\n{context_for_llm}\n\n"
+#     f"Output Format: {output_format_str}"
+# )
+#         prompt_for_llm = (
+#     f"{prompt_instruction_prefix}"
+#     f"Given the following text snippets, analyze the entity/concept {rag_query_phrase} "
+#     f"or the mitochondrial DNA sample in {organism} if these identifiers are not explicitly found. "
+#     f"Identify its **primary associated geographic location**, preferring the most specific available: "
+#     f"first try to determine the exact country; if no country is explicitly mentioned, then provide "
+#     f"the next most specific region, continent, island, or other clear geographic area mentioned. "
+#     f"If no geographic clues at all are present, state 'unknown' for location. "
+#     f"Also, determine if the genetic sample is from a 'modern' (present-day living individual) "
+#     f"or 'ancient' (prehistoric/archaeological) source. "
+#     f"If the text does not specify ancient or archaeological context, assume 'modern'. "
+#     f"Provide only {output_format_str}. "
+#     f"If any information is not explicitly present, use the fallback rules above before defaulting to 'unknown'. "
+#     f"For each non-'unknown' field in {explain_list}, write one sentence explaining how it was inferred from the text (one sentence for each)."
+#     f"\n\nText Snippets:\n{context_for_llm}\n\n"
+#     f"Output Format: {output_format_str}"
+# )
+        prompt_for_llm = (
+    f"{prompt_instruction_prefix}"
+    f"Given the following text snippets, analyze the entity/concept {rag_query_phrase} "
+    f"or the mitochondrial DNA sample in {organism} if these identifiers are not explicitly found. "
+    f"Identify its **primary associated geographic location**, preferring the most specific available: "
+    f"first try to determine the exact country; if no country is explicitly mentioned, then provide "
+    f"the next most specific region, continent, island, or other clear geographic area mentioned. "
+    f"If no geographic clues at all are present, state 'unknown' for location. "
+    f"Also, determine if the genetic sample is from a 'modern' (present-day living individual) "
+    f"or 'ancient' (prehistoric/archaeological) source. "
+    f"If the text does not specify ancient or archaeological context, assume 'modern'. "
+    f"Provide only {output_format_str}. "
+    f"If any information is not explicitly present, use the fallback rules above before defaulting to 'unknown'. "
+    f"For each non-'unknown' field in {explain_list}, write one sentence explaining how it was inferred from the text "
+    f"(one sentence for each). "
+    f"Format your answer so that:\n"
+    f"1. The **first line** contains only the {output_format_str} answer.\n"
+    f"2. The **second line onward** contains the explanations.\n"
+    f"\nText Snippets:\n{context_for_llm}\n\n"
+    f"Output Format Example:\nChina, modern, Daur, Heilongjiang province.\n"
+    f"The text explicitly states \"chinese Daur ethnic group in Heilongjiang province\", indicating the country, "
+    f"the ethnicity, and the specific province. The study is published in a journal, implying research on living "
+    f"individuals, hence modern."
+)
+
+        if model_ai:
+          print("back up to ", model_ai)
+          llm_response_text, model_instance = call_llm_api(prompt_for_llm, model=model_ai)
+        else:
+          print("still 2.5 flash gemini")
+          llm_response_text, model_instance = call_llm_api(prompt_for_llm)
+        print("\n--- DEBUG INFO FOR RAG ---")
+        print("Retrieved Context Sent to LLM (first 500 chars):")
+        print(context_for_llm[:500] + "..." if len(context_for_llm) > 500 else context_for_llm)
+        print("\nRaw LLM Response:")
+        print(llm_response_text)
+        print("--- END DEBUG INFO ---")
+
+        llm_cost = 0
+        if model_instance:
+            try:
+                input_llm_tokens = global_llm_model_for_counting_tokens.count_tokens(prompt_for_llm).total_tokens
+                output_llm_tokens = global_llm_model_for_counting_tokens.count_tokens(llm_response_text).total_tokens
+                print(f"  DEBUG: LLM Input tokens: {input_llm_tokens}")
+                print(f"  DEBUG: LLM Output tokens: {output_llm_tokens}")
+                llm_cost = (input_llm_tokens / 1000) * PRICE_PER_1K_INPUT_LLM + \
+                           (output_llm_tokens / 1000) * PRICE_PER_1K_OUTPUT_LLM
+                print(f"  DEBUG: Estimated LLM cost: ${llm_cost:.6f}")
+            except Exception as e:
+                print(f"  DEBUG: Error counting LLM tokens: {e}")
+                llm_cost = 0
+
+        total_query_cost += current_embedding_cost + llm_cost
+        print(f"  DEBUG: Total estimated cost for this RAG query: ${total_query_cost:.6f}")
+        # Parse the LLM's response based on the Output Format actually used
+        # if output_format_str == "ethnicity, specific_location/unknown": # Targeted RAG output
+        #     extracted_ethnicity,extracted_specific_location = clean_llm_output(llm_response_text, output_format_str)
+        # elif output_format_str == "modern/ancient/unknown, ethnicity, specific_location/unknown":
+        #     extracted_type, extracted_ethnicity,extracted_specific_location=clean_llm_output(llm_response_text, output_format_str) 
+        # else: # Full RAG output (country, type, ethnicity, specific_location)
+        #     extracted_country,extracted_type, extracted_ethnicity,extracted_specific_location=clean_llm_output(llm_response_text, output_format_str) 
+        metadata_list = parse_multi_sample_llm_output(llm_response_text, output_format_str)
+        # merge_metadata = merge_metadata_outputs(metadata_list)
+        # if output_format_str == "country_name, modern/ancient/unknown":
+        #   extracted_country, extracted_type = merge_metadata["country"], merge_metadata["sample_type"]  
+        #   country_explanation,sample_type_explanation = merge_metadata["country_explanation"], merge_metadata["sample_type_explanation"]
+        # elif output_format_str == "modern/ancient/unknown":
+        #   extracted_type = merge_metadata["sample_type"]  
+        #   sample_type_explanation = merge_metadata["sample_type_explanation"]
+        # for the output_format that is not default
+        if output_format_str == "country_name, modern/ancient/unknown":
+          outputs = output_format_str.split(", ")
+          extracted_country, extracted_type = metadata_list[outputs[0]]["answer"], metadata_list[outputs[1]]["answer"]  
+          country_explanation,sample_type_explanation = metadata_list[outputs[0]][outputs[0]+"_explanation"], metadata_list[outputs[1]][outputs[1]+"_explanation"]
+          # extracted_ethnicity, extracted_specific_location = metadata_list[outputs[2]]["answer"], metadata_list[outputs[3]]["answer"]  
+          # ethnicity_explanation, specific_loc_explanation = metadata_list[outputs[2]][outputs[2]+"_explanation"], metadata_list[outputs[3]][outputs[3]+"_explanation"]
+    # 6. Optional: Second LLM call for specific_location from general knowledge if still unknown
+    # if extracted_specific_location == 'unknown':
+    #     # Check if we have enough info to ask general knowledge LLM
+    #     if extracted_country != 'unknown' and extracted_ethnicity != 'unknown':
+    #         print(f"  DEBUG: Specific location still unknown. Querying general knowledge LLM from '{extracted_ethnicity}' and '{extracted_country}'.")
+
+    #         general_knowledge_prompt = (
+    #             f"Based on general knowledge, what is a highly specific location (city or district) "
+    #             f"associated with the ethnicity '{extracted_ethnicity}' in '{extracted_country}'? "
+    #             f"Consider the context of scientific studies on human genetics, if known. "
+    #             f"If no common specific location is known, state 'unknown'. "
+    #             f"Provide only the city or district name, or 'unknown'."
+    #         )
+
+    #         general_llm_response, general_llm_model_instance = call_llm_api(general_knowledge_prompt, model_name='gemini-1.5-flash-latest')
+
+    #         if general_llm_response and general_llm_response.lower().strip() != 'unknown':
+    #             extracted_specific_location = general_llm_response.strip() + " (predicted from general knowledge)"
+    #             # Add cost of this second LLM call
+    #             if general_llm_model_instance:
+    #                 try:
+    #                     gk_input_tokens = general_llm_model_instance.count_tokens(general_knowledge_prompt).total_tokens
+    #                     gk_output_tokens = general_llm_model_instance.count_tokens(general_llm_response).total_tokens
+    #                     gk_cost = (gk_input_tokens / 1000) * PRICE_PER_1K_INPUT_LLM + \
+    #                               (gk_output_tokens / 1000) * PRICE_PER_1K_OUTPUT_LLM
+    #                     print(f"  DEBUG: General Knowledge LLM cost to predict specific location alone: ${gk_cost:.6f}")
+    #                     total_query_cost += gk_cost # Accumulate cost
+    #                 except Exception as e:
+    #                     print(f"  DEBUG: Error counting GK LLM tokens: {e}")
+    #         else:
+    #             print("  DEBUG: General knowledge LLM returned unknown or empty for specific location.")
+    # # 6. Optional: Second LLM call for ethnicity from general knowledge if still unknown
+    # if extracted_ethnicity == 'unknown':
+    #     # Check if we have enough info to ask general knowledge LLM
+    #     if extracted_country != 'unknown' and extracted_specific_location != 'unknown':
+    #         print(f"  DEBUG: Ethnicity still unknown. Querying general knowledge LLM from '{extracted_specific_location}' and '{extracted_country}'.")
+
+    #         general_knowledge_prompt = (
+    #             f"Based on general knowledge, what is a highly ethnicity (population) "
+    #             f"associated with the specific location '{extracted_specific_location}' in '{extracted_country}'? "
+    #             f"Consider the context of scientific studies on human genetics, if known. "
+    #             f"If no common ethnicity is known, state 'unknown'. "
+    #             f"Provide only the ethnicity or popluation name, or 'unknown'."
+    #         )
+
+    #         general_llm_response, general_llm_model_instance = call_llm_api(general_knowledge_prompt, model_name='gemini-1.5-flash-latest')
+
+    #         if general_llm_response and general_llm_response.lower().strip() != 'unknown':
+    #             extracted_ethnicity = general_llm_response.strip() + " (predicted from general knowledge)"
+    #             # Add cost of this second LLM call
+    #             if general_llm_model_instance:
+    #                 try:
+    #                     gk_input_tokens = general_llm_model_instance.count_tokens(general_knowledge_prompt).total_tokens
+    #                     gk_output_tokens = general_llm_model_instance.count_tokens(general_llm_response).total_tokens
+    #                     gk_cost = (gk_input_tokens / 1000) * PRICE_PER_1K_INPUT_LLM + \
+    #                               (gk_output_tokens / 1000) * PRICE_PER_1K_OUTPUT_LLM
+    #                     print(f"  DEBUG: General Knowledge LLM cost to predict ethnicity alone: ${gk_cost:.6f}")
+    #                     total_query_cost += gk_cost # Accumulate cost
+    #                 except Exception as e:
+    #                     print(f"  DEBUG: Error counting GK LLM tokens: {e}")
+    #         else:
+    #             print("  DEBUG: General knowledge LLM returned unknown or empty for ethnicity.")            
+
+
+    #return extracted_country, extracted_type, method_used, extracted_ethnicity, extracted_specific_location, total_query_cost
+    return extracted_country, extracted_type, method_used, country_explanation, sample_type_explanation, total_query_cost

core/mtdna_backend.py

@@ -0,0 +1,426 @@
+import os, tempfile, json, re
+# import io
+
+# from app import send_log
+
+from core.mtdna_classifier import classify_sample_location 
+import core.pipeline
+import pandas as pd
+
+import gspread
+from oauth2client.service_account import ServiceAccountCredentials
+import hashlib
+
+# ✅ Load credentials from Hugging Face secret and ✅ Setup Google Sheets
+creds_dict = json.loads(os.environ["GCP_CREDS_JSON"])
+scope = ['https://spreadsheets.google.com/feeds', 'https://www.googleapis.com/auth/drive']
+creds = ServiceAccountCredentials.from_json_keyfile_dict(creds_dict, scope)
+client = gspread.authorize(creds)
+
+# ✅ Extract accessions from input
+def extract_accessions_from_input(file=None, raw_text=""):
+    # print(f"RAW TEXT RECEIVED: {raw_text}")
+    accessions = []
+    seen = set()
+    if file:
+        try:
+            if file.name.endswith(".csv"):
+                df = pd.read_csv(file)
+            elif file.name.endswith(".xlsx"):
+                df = pd.read_excel(file)
+            else:
+                return [], "Unsupported file format. Please upload CSV or Excel."
+            for acc in df.iloc[:, 0].dropna().astype(str).str.strip():
+                if acc not in seen:
+                    accessions.append(acc)
+                    seen.add(acc)
+        except Exception as e:
+            return [], f"Failed to read file: {e}"
+
+    if raw_text:
+        text_ids = [s.strip() for s in re.split(r"[\n,;\t]", raw_text) if s.strip()]
+        for acc in text_ids:
+            if acc not in seen:
+                accessions.append(acc)
+                seen.add(acc)
+
+    return list(accessions), None
+
+# ✅ Load and save usage count
+def hash_user_id(user_input):
+    return hashlib.sha256(user_input.encode()).hexdigest()
+
+def load_user_usage():
+    try:
+        # ✅ Access user_usage_log sheet1 on Google Sheets
+        sheet = client.open("user_usage_log").sheet1
+        data = sheet.get_all_values()
+        # print("data: ", data)
+        # print("🧪 Raw header row from sheet:", data[0])
+        # print("🧪 Character codes in each header:")
+        # for h in data[0]:
+        #     print([ord(c) for c in h])
+
+        if not data or len(data) < 2:
+            # ph in data[0]:
+        #     print("⚠️ Sheet is empty or missing rows.")
+            return {}
+
+        headers = [h.strip().lower() for h in data[0]]
+        if "email" not in headers or "usage_count" not in headers:
+            # print("❌ Header format incorrect. Must have 'email' and 'usage_count'.")
+            return {}
+            
+        permitted_index = headers.index("permitted_samples") if "permitted_samples" in headers else None
+        df = pd.DataFrame(data[1:], columns=headers)
+
+        usage = {}
+        permitted = {}
+        for _, row in df.iterrows():
+            email = row.get("email", "").strip().lower()
+            try:
+                #count = int(row.get("usage_count", 0))
+                try:
+                    count = int(float(row.get("usage_count", 0)))
+                except Exception:
+                    # print(f"⚠️ Invalid usage_count for {email}: {row.get('usage_count')}")
+                    count = 0
+
+                if email:
+                    usage[email] = count
+                    if permitted_index is not None:
+                        try:
+                            permitted_count = int(float(row.get("permitted_samples", 50)))
+                            permitted[email] = permitted_count
+                        except:
+                            permitted[email] = 50
+                        
+            except ValueError:
+                print(f"⚠️ Invalid usage_count for {email}: {row.get('usage_count')}")
+        return usage, permitted
+
+    except Exception as e:
+        print(f"❌ Error in load_user_usage: {e}")
+        return {}, {}
+
+def save_user_usage(usage_dict):
+    try:
+        # ✅ Access user_usage_log on Google Sheets
+        sheet = client.open("user_usage_log").sheet1
+
+        # Build new df
+        df_new = pd.DataFrame(list(usage_dict.items()), columns=["email", "usage_count"])
+        df_new["email"] = df_new["email"].str.strip().str.lower()
+        df_new["usage_count"] = pd.to_numeric(df_new["usage_count"], errors="coerce").fillna(0).astype(int)
+
+        # Read existing data
+        existing_data = sheet.get_all_values()
+        if existing_data and len(existing_data[0]) >= 2:
+            df_old = pd.DataFrame(existing_data[1:], columns=existing_data[0])
+            df_old["email"] = df_old["email"].str.strip().str.lower()
+            df_old["usage_count"] = pd.to_numeric(df_old["usage_count"], errors="coerce").fillna(0).astype(int)
+        else:
+            df_old = pd.DataFrame(columns=["email", "usage_count"])
+
+        # ✅ Overwrite specific emails only
+        df_old = df_old.set_index("email")
+        for email, count in usage_dict.items():
+            email = email.strip().lower()
+            df_old.loc[email, "usage_count"] = count
+        df_old = df_old.reset_index()
+
+        # Save
+        sheet.clear()
+        sheet.update([df_old.columns.tolist()] + df_old.astype(str).values.tolist())
+        print("✅ Saved user usage to user_usage_log sheet.")
+
+    except Exception as e:
+        print(f"❌ Failed to save user usage to Google Sheets: {e}")
+
+def increment_usage(email: str, count: int = 1):
+    usage, permitted = load_user_usage()
+    email_key = email.strip().lower()
+    #usage[email_key] = usage.get(email_key, 0) + count
+    current = usage.get(email_key, 0)
+    new_value = current + count
+    max_allowed = permitted.get(email_key) or 50
+    usage[email_key] = max(current, new_value)  # ✅ Prevent overwrite with lower
+    # print(f"🧪 increment_usage saving: {email_key=} {current=} + {count=} => {usage[email_key]=}")
+    # print("max allow is: ", max_allowed)
+    save_user_usage(usage)
+    return usage[email_key], max_allowed
+
+# ✅ Save user feedbacks
+def store_feedback_to_google_sheets(accession, answer1, answer2, contact=""):
+    if not answer1.strip() or not answer2.strip():
+        return "⚠️ Please answer both questions before submitting."
+
+    try:
+        # Access feedback_mtdna sheet1 on Google Sheet
+        sheet = client.open("feedback_mtdna").sheet1  # make sure sheet name matches
+
+        # Append feedback
+        sheet.append_row([accession, answer1, answer2, contact])
+        return "✅ Feedback submitted. Thank you!"
+
+    except Exception as e:
+        return f"❌ Error submitting feedback: {e}"
+
+# ✅ save cost by checking the known outputs
+def check_known_output(accession):
+    # print("inside check known output function")
+    try:
+        # ✅ Access known_samples sheet1 on Google Sheet
+        sheet = client.open("known_samples").sheet1
+
+        data = sheet.get_all_values()
+        if not data:
+            # print("⚠️ Google Sheet 'known_samples' is empty.")
+            return None
+
+        df = pd.DataFrame(data[1:], columns=data[0])
+        if "Sample ID" not in df.columns:
+            # print("❌ Column 'Sample ID' not found in Google Sheet.")
+            return None
+        
+        match = re.search(r"\b[A-Z]{2,4}\d{4,}", accession)
+        if match:
+            accession = match.group(0)
+
+        matched = df[df["Sample ID"].str.contains(accession, case=False, na=False)]
+        if not matched.empty:
+            #return matched.iloc[0].to_dict()
+            row = matched.iloc[0]
+            country = row.get("Predicted Country", "").strip().lower()
+            sample_type = row.get("Predicted Sample Type", "").strip().lower()
+
+            if country and country != "unknown" and sample_type and sample_type != "unknown":
+                return row.to_dict()
+            else:
+                # print(f"⚠️ Accession {accession} found but country/sample_type is unknown or empty.")
+                return None
+        else:
+            # print(f"🔍 Accession {accession} not found in known_samples.")
+            return None
+
+    except Exception as e:
+        import traceback
+        # print("❌ Exception occurred during check_known_output:")
+        # traceback.print_exc()
+        return None
+
+# Add a new helper to backend: `filter_unprocessed_accessions()`
+def get_incomplete_accessions(file_path):
+    df = pd.read_excel(file_path)
+
+    incomplete_accessions = []
+    for _, row in df.iterrows():
+        sample_id = str(row.get("Sample ID", "")).strip()
+
+        # Skip if no sample ID
+        if not sample_id:
+            continue
+
+        # Drop the Sample ID and check if the rest is empty
+        other_cols = row.drop(labels=["Sample ID"], errors="ignore")
+        if other_cols.isna().all() or (other_cols.astype(str).str.strip() == "").all():
+            # Extract the accession number from the sample ID using regex
+            match = re.search(r"\b[A-Z]{2,4}\d{4,}", sample_id)
+            if match:
+                incomplete_accessions.append(match.group(0))
+    # print(len(incomplete_accessions))
+    return incomplete_accessions
+
+# Small pipeline wrapper
+def pipeline_classify_sample_location_cached(accession,stop_flag=None, save_df=None):
+    # print("inside pipeline_classify_sample_location_cached, and [accession] is ", [accession])
+    # print("len of save df: ", len(save_df))
+    return pipeline.pipeline_with_gemini([accession],stop_flag=stop_flag, save_df=save_df)    
+
+def summarize_results(accession, stop_flag=None):
+    # Early bail
+    if stop_flag is not None and stop_flag.value:
+        # print(f"🛑 Skipping {accession} before starting.")
+        return []
+    # try cache first
+    cached = check_known_output(accession)
+    if cached:
+        # print(f"✅ Using cached result for {accession}")
+        return cached
+    # only run when nothing in the cache  
+    try:
+        sheet = client.open("known_samples").sheet1
+
+        data = sheet.get_all_values()
+        if not data:
+            # print("⚠️ Google Sheet 'known_samples' is empty.")
+            return None
+
+        save_df = pd.DataFrame(data[1:], columns=data[0])
+        # print("before pipeline, len of save df: ", len(save_df))
+        if stop_flag is not None and stop_flag.value:
+            # print(f"🛑 Skipped {accession} mid-pipeline.")
+            return []
+        else:
+            outputs = pipeline_classify_sample_location_cached(accession, stop_flag, save_df)
+        # outputs = {'KU131308': 
+        #     {'isolate':'BRU18',
+        #     'country': 
+        #         {'brunei': ['ncbi',
+        #                     'rag_llm-The text mentions "BRU18 Brunei Borneo" in a table listing various samples, and it is not described as ancient or archaeological.'
+        #                     ]},
+        #     'sample_type': 
+        #         {'modern': 
+        #                     ['rag_llm-The text mentions "BRU18 Brunei Borneo" in a table listing various samples, and it is not described as ancient or archaeological.'
+        #                     ]},
+        #     'query_cost': 9.754999999999999e-05,
+        #     'time_cost': '24.776 seconds',
+        #     'source': 
+        #         ['https://doi.org/10.1007/s00439-015-1620-z',
+        #         'https://static-content.springer.com/esm/art%3A10.1007%2Fs00439-015-1620-z/MediaObjects/439_2015_1620_MOESM1_ESM.pdf',
+        #         'https://static-content.springer.com/esm/art%3A10.1007%2Fs00439-015-1620-z/MediaObjects/439_2015_1620_MOESM2_ESM.xls']}}
+    except Exception as e:
+        return []#, f"Error: {e}", f"Error: {e}", f"Error: {e}"
+
+    if accession not in outputs:
+        # print("no accession in output ", accession)
+        return []#, "Accession not found in results.", "Accession not found in results.", "Accession not found in results."
+
+    row_score = []
+    rows = []
+    save_rows = []
+    for key in outputs:
+        pred_country, pred_sample, country_explanation, sample_explanation = "unknown","unknown","unknown","unknown" 
+        for section, results in outputs[key].items():
+          if section == "country" or section =="sample_type":
+            pred_output = []#"\n".join(list(results.keys()))
+            output_explanation = ""
+            for result, content in results.items():
+              if len(result) == 0:  result = "unknown"
+              if len(content) == 0: output_explanation = "unknown"
+              else:
+                output_explanation += 'Method: ' + "\nMethod: ".join(content)  + "\n"
+              pred_output.append(result)
+            pred_output = "\n".join(pred_output)    
+            if section == "country":
+              pred_country, country_explanation = pred_output, output_explanation
+            elif section == "sample_type":
+              pred_sample, sample_explanation = pred_output, output_explanation    
+          if outputs[key]["isolate"].lower()!="unknown":
+            label = key + "(Isolate: " + outputs[key]["isolate"] + ")"
+          else: label = key  
+        if len(outputs[key]["source"]) == 0:  outputs[key]["source"] = ["No Links"]
+        # row = {
+        #     "sample_id": label or "unknown",
+        #     "predicted_country": pred_country or "unknown",
+        #     "country_explanation": country_explanation or "unknown",
+        #     "predicted_sample_type":pred_sample or "unknown",
+        #     "sample_type_explanation":sample_explanation or "unknown",
+        #     "sources": "\n".join(outputs[key]["source"]) or "No Links",
+        #     "time_cost": outputs[key]["time_cost"]
+        # }
+        #row_score.append(row)
+        # rows.append(list(row.values()))
+
+        save_row = {
+            "Sample ID": label or "unknown",
+            "Predicted Country": pred_country or "unknown",
+            "Country Explanation": country_explanation or "unknown",
+            "Predicted Sample Type":pred_sample or "unknown",
+            "Sample Type Explanation":sample_explanation or "unknown",
+            "Sources": "\n".join(outputs[key]["source"]) or "No Links",
+            "Query_cost": outputs[key]["query_cost"] or "",
+            "Time cost": outputs[key]["time_cost"] or "",
+            "file_chunk":outputs[key]["file_chunk"] or "",
+            "file_all_output":outputs[key]["file_all_output"] or ""
+        }
+    #     #row_score.append(row)
+        save_rows.append(save_row)
+        
+    return save_rows[0] #, summary, labelAncient_Modern, explain_label
+
+# def run_each_accessions(accession)
+
+# save the batch output in excel file
+def save_to_excel(all_rows, summary_text, flag_text, filename, is_resume=False):
+    df_new = pd.DataFrame(all_rows, columns=[
+        "Sample ID", "Predicted Country", "Country Explanation", 
+        "Predicted Sample Type", "Sample Type Explanation", 
+        "Sources", "Time cost"
+    ])
+
+    if is_resume and os.path.exists(filename):
+        try:
+            df_old = pd.read_excel(filename)
+        except Exception as e:
+            # print(f"⚠️ Warning reading old Excel file: {e}")
+            df_old = pd.DataFrame(columns=df_new.columns)
+
+        # Set index and update existing rows
+        df_old.set_index("Sample ID", inplace=True)
+        df_new.set_index("Sample ID", inplace=True)
+        df_old.update(df_new)
+
+        df_combined = df_old.reset_index()
+    else:
+        # If not resuming or file doesn't exist, just use new rows
+        df_combined = df_new
+
+    # try:
+        df_combined.to_excel(filename, index=False)
+    # except Exception as e:
+        # print(f"❌ Failed to write Excel file {filename}: {e}")
+
+
+# save the batch output in JSON file
+def save_to_json(all_rows, summary_text, flag_text, filename):
+    output_dict = {
+        "Detailed_Results": all_rows#,  # <-- make sure this is a plain list, not a DataFrame
+        # "Summary_Text": summary_text,
+        # "Ancient_Modern_Flag": flag_text
+    }
+
+    # If all_rows is a DataFrame, convert it
+    if isinstance(all_rows, pd.DataFrame):
+        output_dict["Detailed_Results"] = all_rows.to_dict(orient="records")
+
+    with open(filename, "w") as external_file:
+        json.dump(output_dict, external_file, indent=2)
+
+# save the batch output in Text file
+def save_to_txt(all_rows, summary_text, flag_text, filename):
+    if isinstance(all_rows, pd.DataFrame):
+        detailed_results = all_rows.to_dict(orient="records")
+    output = ""
+    #output += ",".join(list(detailed_results[0].keys())) + "\n\n"
+    output += ",".join([str(k) for k in detailed_results[0].keys()]) + "\n\n"
+    for r in detailed_results:
+      output += ",".join([str(v) for v in r.values()]) + "\n\n"
+    with open(filename, "w") as f:
+        f.write("=== Detailed Results ===\n")
+        f.write(output + "\n")
+
+def save_batch_output(all_rows, output_type, summary_text=None, flag_text=None):
+    tmp_dir = tempfile.mkdtemp()
+
+    #html_table = all_rows.value  # assuming this is stored somewhere
+
+    # Parse back to DataFrame
+    #all_rows = pd.read_html(all_rows)[0]  # [0] because read_html returns a list
+    all_rows = pd.read_html(StringIO(all_rows))[0]
+    # print(all_rows)
+
+    if output_type == "Excel":
+        file_path = f"{tmp_dir}/batch_output.xlsx"
+        save_to_excel(all_rows, summary_text, flag_text, file_path)
+    elif output_type == "JSON":
+        file_path = f"{tmp_dir}/batch_output.json"
+        save_to_json(all_rows, summary_text, flag_text, file_path)
+        # print("Done with JSON")
+    elif output_type == "TXT":
+        file_path = f"{tmp_dir}/batch_output.txt"
+        save_to_txt(all_rows, summary_text, flag_text, file_path)
+    else:
+        return gr.update(visible=False)  # invalid option
+    
+    return gr.update(value=file_path, visible=True)

core/mtdna_classifier.py

@@ -0,0 +1,764 @@
+# mtDNA Location Classifier MVP (Google Colab)
+# Accepts accession number → Fetches PubMed ID + isolate name → Gets abstract → Predicts location
+import os
+#import streamlit as st
+import subprocess
+import re
+from Bio import Entrez
+import fitz
+import spacy
+from spacy.cli import download
+# from core.NER.PDF import pdf
+# from core.NER.WordDoc import wordDoc
+# from core.NER.html import extractHTML
+# from core.NER.word2Vec import word2vec
+from transformers import pipeline
+import urllib.parse, requests
+from pathlib import Path
+from core.upgradeClassify import filter_context_for_sample, infer_location_for_sample
+
+# Set your email (required by NCBI Entrez)
+#Entrez.email = "your-email@example.com"
+import nltk
+
+nltk.download("stopwords")
+nltk.download("punkt")
+nltk.download('punkt_tab')
+# Step 1: Get PubMed ID from Accession using EDirect
+from Bio import Entrez, Medline
+import re
+
+Entrez.email = "your_email@example.com"
+
+# --- Helper Functions (Re-organized and Upgraded) ---
+
+def fetch_ncbi_metadata(accession_number):
+    """
+    Fetches metadata directly from NCBI GenBank using Entrez.
+    Includes robust error handling and improved field extraction.
+    Prioritizes location extraction from geo_loc_name, then notes, then other qualifiers.
+    Also attempts to extract ethnicity and sample_type (ancient/modern).
+
+    Args:
+        accession_number (str): The NCBI accession number (e.g., "ON792208").
+
+    Returns:
+        dict: A dictionary containing 'country', 'specific_location', 'ethnicity',
+              'sample_type', 'collection_date', 'isolate', 'title', 'doi', 'pubmed_id'.
+    """
+    Entrez.email = "your.email@example.com" # Required by NCBI, REPLACE WITH YOUR EMAIL
+
+    country = "unknown"
+    specific_location = "unknown"
+    ethnicity = "unknown"
+    sample_type = "unknown"
+    collection_date = "unknown"
+    isolate = "unknown"
+    title = "unknown"
+    doi = "unknown"
+    pubmed_id = None
+    all_feature = "unknown"
+
+    KNOWN_COUNTRIES = [
+        "Afghanistan", "Albania", "Algeria", "Andorra", "Angola", "Antigua and Barbuda", "Argentina", "Armenia", "Australia", "Austria", "Azerbaijan",
+        "Bahamas", "Bahrain", "Bangladesh", "Barbados", "Belarus", "Belgium", "Belize", "Benin", "Bhutan", "Bolivia", "Bosnia and Herzegovina", "Botswana", "Brazil", "Brunei", "Bulgaria", "Burkina Faso", "Burundi",
+        "Cabo Verde", "Cambodia", "Cameroon", "Canada", "Central African Republic", "Chad", "Chile", "China", "Colombia", "Comoros", "Congo (Brazzaville)", "Congo (Kinshasa)", "Costa Rica", "Croatia", "Cuba", "Cyprus", "Czechia",
+        "Denmark", "Djibouti", "Dominica", "Dominican Republic", "Ecuador", "Egypt", "El Salvador", "Equatorial Guinea", "Eritrea", "Estonia", "Eswatini", "Ethiopia",
+        "Fiji", "Finland", "France", "Gabon", "Gambia", "Georgia", "Germany", "Ghana", "Greece", "Grenada", "Guatemala", "Guinea", "Guinea-Bissau", "Guyana",
+        "Haiti", "Honduras", "Hungary", "Iceland", "India", "Indonesia", "Iran", "Iraq", "Ireland", "Israel", "Italy", "Ivory Coast", "Jamaica", "Japan", "Jordan",
+        "Kazakhstan", "Kenya", "Kiribati", "Kosovo", "Kuwait", "Kyrgyzstan", "Laos", "Latvia", "Lebanon", "Lesotho", "Liberia", "Libya", "Liechtenstein", "Lithuania", "Luxembourg",
+        "Madagascar", "Malawi", "Malaysia", "Maldives", "Mali", "Malta", "Marshall Islands", "Mauritania", "Mauritius", "Mexico", "Micronesia", "Moldova", "Monaco", "Mongolia", "Montenegro", "Morocco", "Mozambique", "Myanmar",
+        "Namibia", "Nauru", "Nepal", "Netherlands", "New Zealand", "Nicaragua", "Niger", "Nigeria", "North Korea", "North Macedonia", "Norway", "Oman",
+        "Pakistan", "Palau", "Palestine", "Panama", "Papua New Guinea", "Paraguay", "Peru", "Philippines", "Poland", "Portugal", "Qatar", "Romania", "Russia", "Rwanda",
+        "Saint Kitts and Nevis", "Saint Lucia", "Saint Vincent and the Grenadines", "Samoa", "San Marino", "Sao Tome and Principe", "Saudi Arabia", "Senegal", "Serbia", "Seychelles", "Sierra Leone", "Singapore", "Slovakia", "Slovenia", "Solomon Islands", "Somalia", "South Africa", "South Korea", "South Sudan", "Spain", "Sri Lanka", "Sudan", "Suriname", "Sweden", "Switzerland", "Syria",
+        "Taiwan", "Tajikistan", "Tanzania", "Thailand", "Timor-Leste", "Togo", "Tonga", "Trinidad and Tobago", "Tunisia", "Turkey", "Turkmenistan", "Tuvalu",
+        "Uganda", "Ukraine", "United Arab Emirates", "United Kingdom", "United States", "Uruguay", "Uzbekistan", "Vanuatu", "Vatican City", "Venezuela", "Vietnam",
+        "Yemen", "Zambia", "Zimbabwe"
+    ]
+    COUNTRY_PATTERN = re.compile(r'\b(' + '|'.join(re.escape(c) for c in KNOWN_COUNTRIES) + r')\b', re.IGNORECASE)
+
+    try:
+        handle = Entrez.efetch(db="nucleotide", id=str(accession_number), rettype="gb", retmode="xml")
+        record = Entrez.read(handle)
+        handle.close()
+
+        gb_seq = None
+        # Validate record structure: It should be a list with at least one element (a dict)
+        if isinstance(record, list) and len(record) > 0:
+            if isinstance(record[0], dict):
+                gb_seq = record[0]
+            else:
+                print(f"Warning: record[0] is not a dictionary for {accession_number}. Type: {type(record[0])}")
+        else:
+            print(f"Warning: No valid record or empty record list from NCBI for {accession_number}.")
+
+        # If gb_seq is still None, return defaults
+        if gb_seq is None:
+            return {"country": "unknown",
+                "specific_location": "unknown",
+                "ethnicity": "unknown",
+                "sample_type": "unknown",
+                "collection_date": "unknown",
+                "isolate": "unknown",
+                "title": "unknown",
+                "doi": "unknown",
+                "pubmed_id": None,
+                "all_features": "unknown"}
+
+
+        # If gb_seq is valid, proceed with extraction
+        collection_date = gb_seq.get("GBSeq_create-date","unknown")
+
+        references = gb_seq.get("GBSeq_references", [])
+        for ref in references:
+            if not pubmed_id:
+                pubmed_id = ref.get("GBReference_pubmed",None)
+            if title == "unknown":
+                title = ref.get("GBReference_title","unknown")
+            for xref in ref.get("GBReference_xref", []):
+                if xref.get("GBXref_dbname") == "doi":
+                    doi = xref.get("GBXref_id")
+                    break
+
+        features = gb_seq.get("GBSeq_feature-table", [])
+
+        context_for_flagging = "" # Accumulate text for ancient/modern detection
+        features_context = ""
+        for feature in features:
+            if feature.get("GBFeature_key") == "source":
+                feature_context = ""
+                qualifiers = feature.get("GBFeature_quals", [])
+                found_country = "unknown"
+                found_specific_location = "unknown"
+                found_ethnicity = "unknown"
+
+                temp_geo_loc_name = "unknown"
+                temp_note_origin_locality = "unknown"
+                temp_country_qual = "unknown"
+                temp_locality_qual = "unknown"
+                temp_collection_location_qual = "unknown"
+                temp_isolation_source_qual = "unknown"
+                temp_env_sample_qual = "unknown"
+                temp_pop_qual = "unknown"
+                temp_organism_qual = "unknown"
+                temp_specimen_qual = "unknown"
+                temp_strain_qual = "unknown"
+
+                for qual in qualifiers:
+                    qual_name = qual.get("GBQualifier_name")
+                    qual_value = qual.get("GBQualifier_value")
+                    feature_context += qual_name + ": " + qual_value +"\n"
+                    if qual_name == "collection_date":
+                        collection_date = qual_value
+                    elif qual_name == "isolate":
+                        isolate = qual_value
+                    elif qual_name == "population":
+                        temp_pop_qual = qual_value
+                    elif qual_name == "organism":
+                        temp_organism_qual = qual_value
+                    elif qual_name == "specimen_voucher" or qual_name == "specimen":
+                        temp_specimen_qual = qual_value
+                    elif qual_name == "strain":
+                        temp_strain_qual = qual_value
+                    elif qual_name == "isolation_source":
+                        temp_isolation_source_qual = qual_value
+                    elif qual_name == "environmental_sample":
+                        temp_env_sample_qual = qual_value
+
+                    if qual_name == "geo_loc_name": temp_geo_loc_name = qual_value
+                    elif qual_name == "note":
+                        if qual_value.startswith("origin_locality:"):
+                            temp_note_origin_locality = qual_value
+                        context_for_flagging += qual_value + " " # Capture all notes for flagging  
+                    elif qual_name == "country": temp_country_qual = qual_value
+                    elif qual_name == "locality": temp_locality_qual = qual_value
+                    elif qual_name == "collection_location": temp_collection_location_qual = qual_value
+
+
+                # --- Aggregate all relevant info into context_for_flagging ---
+                context_for_flagging += f" {isolate} {temp_isolation_source_qual} {temp_specimen_qual} {temp_strain_qual} {temp_organism_qual} {temp_geo_loc_name} {temp_collection_location_qual} {temp_env_sample_qual}"
+                context_for_flagging = context_for_flagging.strip()
+                
+                # --- Determine final country and specific_location based on priority ---
+                if temp_geo_loc_name != "unknown":
+                    parts = [p.strip() for p in temp_geo_loc_name.split(':')]
+                    if len(parts) > 1: 
+                      found_specific_location = parts[-1]; found_country = parts[0]
+                    else: found_country = temp_geo_loc_name; found_specific_location = "unknown"
+                elif temp_note_origin_locality != "unknown":
+                    match = re.search(r"origin_locality:\s*(.*)", temp_note_origin_locality, re.IGNORECASE)
+                    if match:
+                        location_string = match.group(1).strip()
+                        parts = [p.strip() for p in location_string.split(':')]
+                        if len(parts) > 1: found_country = parts[-1]; found_specific_location = parts[0]
+                        else: found_country = location_string; found_specific_location = "unknown"
+                elif temp_locality_qual != "unknown":
+                    found_country_match = COUNTRY_PATTERN.search(temp_locality_qual)
+                    if found_country_match: found_country = found_country_match.group(1); temp_loc = re.sub(re.escape(found_country), '', temp_locality_qual, flags=re.IGNORECASE).strip().replace(',', '').replace(':', '').replace(';', '').strip(); found_specific_location = temp_loc if temp_loc else "unknown"
+                    else: found_specific_location = temp_locality_qual; found_country = "unknown"
+                elif temp_collection_location_qual != "unknown":
+                    found_country_match = COUNTRY_PATTERN.search(temp_collection_location_qual)
+                    if found_country_match: found_country = found_country_match.group(1); temp_loc = re.sub(re.escape(found_country), '', temp_collection_location_qual, flags=re.IGNORECASE).strip().replace(',', '').replace(':', '').replace(';', '').strip(); found_specific_location = temp_loc if temp_loc else "unknown"
+                    else: found_specific_location = temp_collection_location_qual; found_country = "unknown"
+                elif temp_isolation_source_qual != "unknown":
+                    found_country_match = COUNTRY_PATTERN.search(temp_isolation_source_qual)
+                    if found_country_match: found_country = found_country_match.group(1); temp_loc = re.sub(re.escape(found_country), '', temp_isolation_source_qual, flags=re.IGNORECASE).strip().replace(',', '').replace(':', '').replace(';', '').strip(); found_specific_location = temp_loc if temp_loc else "unknown"
+                    else: found_specific_location = temp_isolation_source_qual; found_country = "unknown"
+                elif temp_env_sample_qual != "unknown":
+                    found_country_match = COUNTRY_PATTERN.search(temp_env_sample_qual)
+                    if found_country_match: found_country = found_country_match.group(1); temp_loc = re.sub(re.escape(found_country), '', temp_env_sample_qual, flags=re.IGNORECASE).strip().replace(',', '').replace(':', '').replace(';', '').strip(); found_specific_location = temp_loc if temp_loc else "unknown"
+                    else: found_specific_location = temp_env_sample_qual; found_country = "unknown"
+                if found_country == "unknown" and temp_country_qual != "unknown":
+                     found_country_match = COUNTRY_PATTERN.search(temp_country_qual)
+                     if found_country_match: found_country = found_country_match.group(1)
+
+                country = found_country
+                specific_location = found_specific_location
+                # --- Determine final ethnicity ---
+                if temp_pop_qual != "unknown":
+                    found_ethnicity = temp_pop_qual
+                elif isolate != "unknown" and re.fullmatch(r'[A-Za-z\s\-]+', isolate) and get_country_from_text(isolate) == "unknown":
+                     found_ethnicity = isolate
+                elif context_for_flagging != "unknown": # Use the broader context for ethnicity patterns
+                    eth_match = re.search(r'(?:population|ethnicity|isolate source):\s*([A-Za-z\s\-]+)', context_for_flagging, re.IGNORECASE)
+                    if eth_match:
+                        found_ethnicity = eth_match.group(1).strip()
+
+                ethnicity = found_ethnicity
+
+                # --- Determine sample_type (ancient/modern) ---
+                if context_for_flagging:
+                    sample_type, explain = detect_ancient_flag(context_for_flagging)
+                features_context += feature_context + "\n"
+                break
+
+        if specific_location != "unknown" and specific_location.lower() == country.lower():
+            specific_location = "unknown"
+        if not features_context:  features_context = "unknown"    
+        return {"country": country.lower(),
+                "specific_location": specific_location.lower(),
+                "ethnicity": ethnicity.lower(),
+                "sample_type": sample_type.lower(),
+                "collection_date": collection_date,
+                "isolate": isolate,
+                "title": title,
+                "doi": doi,
+                "pubmed_id": pubmed_id,
+                "all_features": features_context}
+
+    except:
+        print(f"Error fetching NCBI data for {accession_number}")
+        return {"country": "unknown",
+                "specific_location": "unknown",
+                "ethnicity": "unknown",
+                "sample_type": "unknown",
+                "collection_date": "unknown",
+                "isolate": "unknown",
+                "title": "unknown",
+                "doi": "unknown",
+                "pubmed_id": None,
+                "all_features": "unknown"}
+
+# --- Helper function for country matching (re-defined from main code to be self-contained) ---
+_country_keywords = {
+    "thailand": "Thailand", "laos": "Laos", "cambodia": "Cambodia", "myanmar": "Myanmar",
+    "philippines": "Philippines", "indonesia": "Indonesia", "malaysia": "Malaysia",
+    "china": "China", "chinese": "China", "india": "India", "taiwan": "Taiwan",
+    "vietnam": "Vietnam", "russia": "Russia", "siberia": "Russia", "nepal": "Nepal",
+    "japan": "Japan", "sumatra": "Indonesia", "borneu": "Indonesia",
+    "yunnan": "China", "tibet": "China", "northern mindanao": "Philippines",
+    "west malaysia": "Malaysia", "north thailand": "Thailand", "central thailand": "Thailand",
+    "northeast thailand": "Thailand", "east myanmar": "Myanmar", "west thailand": "Thailand",
+    "central india": "India", "east india": "India", "northeast india": "India",
+    "south sibera": "Russia", "mongolia": "China", "beijing": "China", "south korea": "South Korea",
+    "north asia": "unknown", "southeast asia": "unknown", "east asia": "unknown"
+}
+
+def get_country_from_text(text):
+    text_lower = text.lower()
+    for keyword, country in _country_keywords.items():
+        if keyword in text_lower:
+            return country
+    return "unknown"
+# The result will be seen as manualLink for the function get_paper_text
+# def search_google_custom(query, max_results=3):
+#   # query should be the title from ncbi or paper/source title
+#     GOOGLE_CSE_API_KEY = os.environ["GOOGLE_CSE_API_KEY"]
+#     GOOGLE_CSE_CX = os.environ["GOOGLE_CSE_CX"]
+#     endpoint = os.environ["SEARCH_ENDPOINT"]
+#     params = {
+#         "key": GOOGLE_CSE_API_KEY,
+#         "cx": GOOGLE_CSE_CX,
+#         "q": query,
+#         "num": max_results
+#     }
+#     try:
+#         response = requests.get(endpoint, params=params)
+#         if response.status_code == 429:
+#             print("Rate limit hit. Try again later.")
+#             return []
+#         response.raise_for_status()
+#         data = response.json().get("items", [])
+#         return [item.get("link") for item in data if item.get("link")]
+#     except Exception as e:
+#         print("Google CSE error:", e)
+#         return []
+
+def search_google_custom(query, max_results=3):
+  # query should be the title from ncbi or paper/source title
+    GOOGLE_CSE_API_KEY = os.environ["GOOGLE_CSE_API_KEY"]
+    GOOGLE_CSE_CX = os.environ["GOOGLE_CSE_CX"]
+    endpoint = os.environ["SEARCH_ENDPOINT"]
+    params = {
+        "key": GOOGLE_CSE_API_KEY,
+        "cx": GOOGLE_CSE_CX,
+        "q": query,
+        "num": max_results
+    }
+    try:
+        response = requests.get(endpoint, params=params)
+        if response.status_code == 429:
+            print("Rate limit hit. Try again later.")
+            print("try with back up account")
+            try: 
+              return search_google_custom_backup(query, max_results)
+            except:
+              return []
+        response.raise_for_status()
+        data = response.json().get("items", [])
+        return [item.get("link") for item in data if item.get("link")]
+    except Exception as e:
+        print("Google CSE error:", e)
+        return []
+
+def search_google_custom_backup(query, max_results=3):
+  # query should be the title from ncbi or paper/source title
+    GOOGLE_CSE_API_KEY = os.environ["GOOGLE_CSE_API_KEY_BACKUP"]
+    GOOGLE_CSE_CX = os.environ["GOOGLE_CSE_CX_BACKUP"]
+    endpoint = os.environ["SEARCH_ENDPOINT"]
+    params = {
+        "key": GOOGLE_CSE_API_KEY,
+        "cx": GOOGLE_CSE_CX,
+        "q": query,
+        "num": max_results
+    }
+    try:
+        response = requests.get(endpoint, params=params)
+        if response.status_code == 429:
+            print("Rate limit hit. Try again later.")
+            return []
+        response.raise_for_status()
+        data = response.json().get("items", [])
+        return [item.get("link") for item in data if item.get("link")]
+    except Exception as e:
+        print("Google CSE error:", e)
+        return []
+# Step 3: Extract Text: Get the paper (html text), sup. materials (pdf, doc, excel) and do text-preprocessing
+# Step 3.1: Extract Text
+# sub: download excel file
+def download_excel_file(url, save_path="temp.xlsx"):
+    if "view.officeapps.live.com" in url:
+        parsed_url = urllib.parse.parse_qs(urllib.parse.urlparse(url).query)
+        real_url = urllib.parse.unquote(parsed_url["src"][0])
+        response = requests.get(real_url)
+        with open(save_path, "wb") as f:
+            f.write(response.content)
+        return save_path
+    elif url.startswith("http") and (url.endswith(".xls") or url.endswith(".xlsx")):
+        response = requests.get(url)
+        response.raise_for_status()  # Raises error if download fails
+        with open(save_path, "wb") as f:
+            f.write(response.content)
+        return save_path
+    else:
+        print("URL must point directly to an .xls or .xlsx file\n or it already downloaded.")
+        return url
+def get_paper_text(doi,id,manualLinks=None):
+  # create the temporary folder to contain the texts
+  folder_path = Path("data/"+str(id))
+  if not folder_path.exists():
+      cmd = f'mkdir data/{id}'
+      result = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+      print("data/"+str(id) +" created.")
+  else:
+      print("data/"+str(id) +" already exists.")
+  saveLinkFolder = "data/"+id
+
+  link = 'https://doi.org/' + doi
+  '''textsToExtract = { "doiLink":"paperText"
+                        "file1.pdf":"text1",
+                        "file2.doc":"text2",
+                        "file3.xlsx":excelText3'''
+  textsToExtract = {}
+  # get the file to create listOfFile for each id
+  html = extractHTML.HTML("",link)
+  jsonSM = html.getSupMaterial()
+  text = ""
+  links  = [link] + sum((jsonSM[key] for key in jsonSM),[])
+  if manualLinks != None:
+    links += manualLinks
+  for l in links:
+    # get the main paper
+    name = l.split("/")[-1]
+    file_path = folder_path / name
+    if l == link:
+      text = html.getListSection()
+      textsToExtract[link] = text
+    elif l.endswith(".pdf"):
+      if file_path.is_file():
+          l = saveLinkFolder + "/" + name
+          print("File exists.")
+      p = pdf.PDF(l,saveLinkFolder,doi)
+      f = p.openPDFFile()
+      pdf_path = saveLinkFolder + "/" + l.split("/")[-1]
+      doc = fitz.open(pdf_path)
+      text = "\n".join([page.get_text() for page in doc])
+      textsToExtract[l] = text
+    elif l.endswith(".doc") or l.endswith(".docx"):
+      d = wordDoc.wordDoc(l,saveLinkFolder)
+      text = d.extractTextByPage()
+      textsToExtract[l] = text
+    elif l.split(".")[-1].lower() in "xlsx":
+      wc = word2vec.word2Vec()
+      # download excel file if it not downloaded yet
+      savePath = saveLinkFolder +"/"+ l.split("/")[-1]
+      excelPath = download_excel_file(l, savePath)
+      corpus = wc.tableTransformToCorpusText([],excelPath)
+      text = ''
+      for c in corpus:
+        para = corpus[c]
+        for words in para:
+          text += " ".join(words)
+      textsToExtract[l] = text
+  # delete folder after finishing getting text
+  #cmd = f'rm -r data/{id}'
+  #result = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+  return textsToExtract
+# Step 3.2: Extract context
+def extract_context(text, keyword, window=500):
+    # firstly try accession number
+    idx = text.find(keyword)
+    if idx == -1:
+        return "Sample ID not found."
+    return text[max(0, idx-window): idx+window]
+def extract_relevant_paragraphs(text, accession, keep_if=None, isolate=None):
+    if keep_if is None:
+        keep_if = ["sample", "method", "mtdna", "sequence", "collected", "dataset", "supplementary", "table"]
+
+    outputs = ""
+    text = text.lower()
+
+    # If isolate is provided, prioritize paragraphs that mention it
+    # If isolate is provided, prioritize paragraphs that mention it
+    if accession and accession.lower() in text:
+        if extract_context(text, accession.lower(), window=700) != "Sample ID not found.":
+            outputs += extract_context(text, accession.lower(), window=700)       
+    if isolate and isolate.lower() in text:
+        if extract_context(text, isolate.lower(), window=700) != "Sample ID not found.":
+            outputs += extract_context(text, isolate.lower(), window=700)
+    for keyword in keep_if:
+        para = extract_context(text, keyword)
+        if para and para not in outputs:
+            outputs += para + "\n"
+    return outputs
+# Step 4: Classification for now (demo purposes)
+# 4.1: Using a HuggingFace model (question-answering)
+def infer_fromQAModel(context, question="Where is the mtDNA sample from?"):
+    try:
+        qa = pipeline("question-answering", model="distilbert-base-uncased-distilled-squad")
+        result = qa({"context": context, "question": question})
+        return result.get("answer", "Unknown")
+    except Exception as e:
+        return f"Error: {str(e)}"
+
+# 4.2: Infer from haplogroup
+# Load pre-trained spaCy model for NER
+try:
+    nlp = spacy.load("en_core_web_sm")
+except OSError:
+    download("en_core_web_sm")
+    nlp = spacy.load("en_core_web_sm")
+
+# Define the haplogroup-to-region mapping (simple rule-based)
+import csv
+
+def load_haplogroup_mapping(csv_path):
+    mapping = {}
+    with open(csv_path) as f:
+        reader = csv.DictReader(f)
+        for row in reader:
+            mapping[row["haplogroup"]] = [row["region"],row["source"]]
+    return mapping
+
+# Function to extract haplogroup from the text
+def extract_haplogroup(text):
+    match = re.search(r'\bhaplogroup\s+([A-Z][0-9a-z]*)\b', text)
+    if match:
+        submatch = re.match(r'^[A-Z][0-9]*', match.group(1))
+        if submatch:
+            return submatch.group(0)
+        else:
+            return match.group(1)  # fallback
+    fallback = re.search(r'\b([A-Z][0-9a-z]{1,5})\b', text)
+    if fallback:
+        return fallback.group(1)
+    return None
+
+
+# Function to extract location based on NER
+def extract_location(text):
+    doc = nlp(text)
+    locations = []
+    for ent in doc.ents:
+        if ent.label_ == "GPE":  # GPE = Geopolitical Entity (location)
+            locations.append(ent.text)
+    return locations
+
+# Function to infer location from haplogroup
+def infer_location_from_haplogroup(haplogroup):
+  haplo_map = load_haplogroup_mapping("data/haplogroup_regions_extended.csv")
+  return haplo_map.get(haplogroup, ["Unknown","Unknown"])
+
+# Function to classify the mtDNA sample
+def classify_mtDNA_sample_from_haplo(text):
+    # Extract haplogroup
+    haplogroup = extract_haplogroup(text)
+    # Extract location based on NER
+    locations = extract_location(text)
+    # Infer location based on haplogroup
+    inferred_location, sourceHaplo = infer_location_from_haplogroup(haplogroup)[0],infer_location_from_haplogroup(haplogroup)[1]
+    return {
+        "source":sourceHaplo,
+        "locations_found_in_context": locations,
+        "haplogroup": haplogroup,
+        "inferred_location": inferred_location
+
+    }
+# 4.3 Get from available NCBI
+def infer_location_fromNCBI(accession):
+    try:
+        handle = Entrez.efetch(db="nuccore", id=accession, rettype="medline", retmode="text")
+        text = handle.read()
+        handle.close()
+        match = re.search(r'/(geo_loc_name|country|location)\s*=\s*"([^"]+)"', text)
+        if match:
+            return match.group(2), match.group(0)  # This is the value like "Brunei"
+        return "Not found", "Not found"
+
+    except Exception as e:
+        print("❌ Entrez error:", e)
+        return "Not found", "Not found"
+
+### ANCIENT/MODERN FLAG
+from Bio import Entrez
+import re
+
+def flag_ancient_modern(accession, textsToExtract, isolate=None):
+    """
+    Try to classify a sample as Ancient or Modern using:
+    1. NCBI accession (if available)
+    2. Supplementary text or context fallback
+    """
+    context = ""
+    label, explain = "", ""
+
+    try:
+        # Check if we can fetch metadata from NCBI using the accession
+        handle = Entrez.efetch(db="nuccore", id=accession, rettype="medline", retmode="text")
+        text = handle.read()
+        handle.close()
+
+        isolate_source = re.search(r'/(isolation_source)\s*=\s*"([^"]+)"', text)
+        if isolate_source:
+            context += isolate_source.group(0) + " "
+
+        specimen = re.search(r'/(specimen|specimen_voucher)\s*=\s*"([^"]+)"', text)
+        if specimen:
+            context += specimen.group(0) + " "
+
+        if context.strip():
+            label, explain = detect_ancient_flag(context)
+            if label!="Unknown":
+              return label, explain + " from NCBI\n(" + context + ")"
+
+        # If no useful NCBI metadata, check supplementary texts
+        if textsToExtract:
+            labels = {"modern": [0, ""], "ancient": [0, ""], "unknown": 0}
+
+            for source in textsToExtract:
+                text_block = textsToExtract[source]
+                context = extract_relevant_paragraphs(text_block, accession, isolate=isolate)  # Reduce to informative paragraph(s)
+                label, explain = detect_ancient_flag(context)
+
+                if label == "Ancient":
+                    labels["ancient"][0] += 1
+                    labels["ancient"][1] += f"{source}:\n{explain}\n\n"
+                elif label == "Modern":
+                    labels["modern"][0] += 1
+                    labels["modern"][1] += f"{source}:\n{explain}\n\n"
+                else:
+                    labels["unknown"] += 1
+
+            if max(labels["modern"][0],labels["ancient"][0]) > 0:
+                if labels["modern"][0] > labels["ancient"][0]:
+                    return "Modern", labels["modern"][1]
+                else:
+                    return "Ancient", labels["ancient"][1]
+            else:
+              return "Unknown", "No strong keywords detected"
+        else:
+            print("No DOI or PubMed ID available for inference.")
+            return "", ""
+
+    except Exception as e:
+        print("Error:", e)
+        return "", ""
+
+
+def detect_ancient_flag(context_snippet):
+    context = context_snippet.lower()
+
+    ancient_keywords = [
+        "ancient", "archaeological", "prehistoric", "neolithic", "mesolithic", "paleolithic",
+        "bronze age", "iron age", "burial", "tomb", "skeleton", "14c", "radiocarbon", "carbon dating",
+        "postmortem damage", "udg treatment", "adna", "degradation", "site", "excavation",
+        "archaeological context", "temporal transect", "population replacement", "cal bp", "calbp", "carbon dated"
+    ]
+
+    modern_keywords = [
+        "modern", "hospital", "clinical", "consent","blood","buccal","unrelated", "blood sample","buccal sample","informed consent", "donor", "healthy", "patient",
+        "genotyping", "screening", "medical", "cohort", "sequencing facility", "ethics approval",
+        "we analysed", "we analyzed", "dataset includes", "new sequences", "published data",
+        "control cohort", "sink population", "genbank accession", "sequenced", "pipeline", 
+        "bioinformatic analysis", "samples from", "population genetics", "genome-wide data", "imr collection"
+    ]
+
+    ancient_hits = [k for k in ancient_keywords if k in context]
+    modern_hits = [k for k in modern_keywords if k in context]
+
+    if ancient_hits and not modern_hits:
+        return "Ancient", f"Flagged as ancient due to keywords: {', '.join(ancient_hits)}"
+    elif modern_hits and not ancient_hits:
+        return "Modern", f"Flagged as modern due to keywords: {', '.join(modern_hits)}"
+    elif ancient_hits and modern_hits:
+        if len(ancient_hits) >= len(modern_hits):
+            return "Ancient", f"Mixed context, leaning ancient due to: {', '.join(ancient_hits)}"
+        else:
+            return "Modern", f"Mixed context, leaning modern due to: {', '.join(modern_hits)}"
+    
+    # Fallback to QA
+    answer = infer_fromQAModel(context, question="Are the mtDNA samples ancient or modern? Explain why.")
+    if answer.startswith("Error"):
+        return "Unknown", answer
+    if "ancient" in answer.lower():
+        return "Ancient", f"Leaning ancient based on QA: {answer}"
+    elif "modern" in answer.lower():
+        return "Modern", f"Leaning modern based on QA: {answer}"
+    else:
+        return "Unknown", f"No strong keywords or QA clues. QA said: {answer}"
+
+# STEP 5: Main pipeline: accession -> 1. get pubmed id and isolate -> 2. get doi -> 3. get text -> 4. prediction -> 5. output: inferred location + explanation + confidence score
+def classify_sample_location(accession):
+  outputs = {}
+  keyword, context, location, qa_result, haplo_result = "", "", "", "", ""
+  # Step 1: get pubmed id and isolate
+  pubmedID, isolate = get_info_from_accession(accession)
+  '''if not pubmedID:
+    return {"error": f"Could not retrieve PubMed ID for accession {accession}"}'''
+  if not isolate:
+    isolate = "UNKNOWN_ISOLATE"
+  # Step 2: get doi
+  doi = get_doi_from_pubmed_id(pubmedID)
+  '''if not doi:
+    return {"error": "DOI not found for this accession. Cannot fetch paper or context."}'''
+  # Step 3: get text
+  '''textsToExtract = { "doiLink":"paperText"
+                        "file1.pdf":"text1",
+                        "file2.doc":"text2",
+                        "file3.xlsx":excelText3'''
+  if doi and pubmedID:                      
+    textsToExtract = get_paper_text(doi,pubmedID)
+  else: textsToExtract = {}  
+  '''if not textsToExtract:
+    return {"error": f"No texts extracted for DOI {doi}"}'''
+  if isolate not in [None, "UNKNOWN_ISOLATE"]:
+    label, explain = flag_ancient_modern(accession,textsToExtract,isolate)
+  else: 
+    label, explain = flag_ancient_modern(accession,textsToExtract)  
+  # Step 4: prediction
+  outputs[accession] = {}
+  outputs[isolate] = {}
+  # 4.0 Infer from NCBI
+  location, outputNCBI = infer_location_fromNCBI(accession)
+  NCBI_result = {
+      "source": "NCBI",
+      "sample_id": accession,
+      "predicted_location": location,
+      "context_snippet": outputNCBI}
+  outputs[accession]["NCBI"]= {"NCBI": NCBI_result}
+  if textsToExtract:
+    long_text = ""
+    for key in textsToExtract:
+      text = textsToExtract[key]
+      # try accession number first
+      outputs[accession][key] = {}
+      keyword = accession
+      context = extract_context(text, keyword, window=500)
+      # 4.1: Using a HuggingFace model (question-answering)
+      location = infer_fromQAModel(context, question=f"Where is the mtDNA sample {keyword} from?")
+      qa_result = {
+          "source": key,
+          "sample_id": keyword,
+          "predicted_location": location,
+          "context_snippet": context
+      }
+      outputs[keyword][key]["QAModel"] = qa_result
+      # 4.2: Infer from haplogroup
+      haplo_result = classify_mtDNA_sample_from_haplo(context)
+      outputs[keyword][key]["haplogroup"] = haplo_result
+      # try isolate
+      keyword = isolate
+      outputs[isolate][key] = {}
+      context = extract_context(text, keyword, window=500)
+      # 4.1.1: Using a HuggingFace model (question-answering)
+      location = infer_fromQAModel(context, question=f"Where is the mtDNA sample {keyword} from?")
+      qa_result = {
+          "source": key,
+          "sample_id": keyword,
+          "predicted_location": location,
+          "context_snippet": context
+      }
+      outputs[keyword][key]["QAModel"] = qa_result
+      # 4.2.1: Infer from haplogroup
+      haplo_result = classify_mtDNA_sample_from_haplo(context)
+      outputs[keyword][key]["haplogroup"] = haplo_result
+      # add long text
+      long_text += text + ". \n"
+    # 4.3: UpgradeClassify
+    # try sample_id as accession number
+    sample_id = accession
+    if sample_id:
+      filtered_context = filter_context_for_sample(sample_id.upper(), long_text, window_size=1)
+      locations = infer_location_for_sample(sample_id.upper(), filtered_context)
+      if locations!="No clear location found in top matches":
+        outputs[sample_id]["upgradeClassifier"] = {}
+        outputs[sample_id]["upgradeClassifier"]["upgradeClassifier"] = {
+          "source": "From these sources combined: "+ ", ".join(list(textsToExtract.keys())),
+          "sample_id": sample_id,
+          "predicted_location": ", ".join(locations),
+          "context_snippep": "First 1000 words: \n"+ filtered_context[:1000]
+        }
+    # try sample_id as isolate name
+    sample_id = isolate
+    if sample_id:
+      filtered_context = filter_context_for_sample(sample_id.upper(), long_text, window_size=1)
+      locations = infer_location_for_sample(sample_id.upper(), filtered_context)
+      if locations!="No clear location found in top matches":
+        outputs[sample_id]["upgradeClassifier"] = {}
+        outputs[sample_id]["upgradeClassifier"]["upgradeClassifier"] = {
+          "source": "From these sources combined: "+ ", ".join(list(textsToExtract.keys())),
+          "sample_id": sample_id,
+          "predicted_location": ", ".join(locations),
+          "context_snippep": "First 1000 words: \n"+ filtered_context[:1000]
+        }
+  return outputs, label, explain

core/pipeline.py

@@ -0,0 +1,793 @@
+# test1: MJ17 direct
+# test2: "A1YU101" thailand cross-ref
+# test3: "EBK109" thailand cross-ref
+# test4: "OQ731952"/"BST115" for search query title: "South Asian maternal and paternal lineages in southern Thailand and"
+import os, io, time, re, json
+import pandas as pd
+import subprocess
+import multiprocessing
+from pathlib import Path
+from typing import Any, Dict, List, Optional
+
+
+import google.generativeai as genai
+
+
+# Google Drive (optional)
+from google.oauth2.service_account import Credentials
+from googleapiclient.discovery import build
+from googleapiclient.http import MediaFileUpload, MediaIoBaseDownload
+
+import gspread
+from oauth2client.service_account import ServiceAccountCredentials
+
+# ---- core modules (must exist in your project) ----
+import core.mtdna_classifier as mtdna_classifier
+import core.data_preprocess as data_preprocess
+import core.model as model
+import core.smart_fallback as smart_fallback
+import core.standardize_location as standardize_location
+from core.NER.html import extractHTML
+from core.drive_utils import *
+
+# def run_with_timeout(func, args=(), kwargs={}, timeout=20):
+#     """
+#     Runs `func` with timeout in seconds. Kills if it exceeds.
+#     Returns: (success, result or None)
+#     """
+#     def wrapper(q, *args, **kwargs):
+#         try:
+#             q.put(func(*args, **kwargs))
+#         except Exception as e:
+#             q.put(e)
+
+#     q = multiprocessing.Queue()
+#     p = multiprocessing.Process(target=wrapper, args=(q, *args), kwargs=kwargs)
+#     p.start()
+#     p.join(timeout)
+
+#     if p.is_alive():
+#         p.terminate()
+#         p.join()
+#         print(f"⏱️ Timeout exceeded ({timeout} sec) — function killed.")
+#         return False, None
+#     else:
+#         result = q.get()
+#         if isinstance(result, Exception):
+#             raise result
+#         return True, result
+# def run_with_timeout(func, args=(), kwargs={}, timeout=30):
+#     import concurrent.futures
+#     with concurrent.futures.ThreadPoolExecutor(max_workers=1) as executor:
+#         future = executor.submit(func, *args, **kwargs)
+#         try:
+#             return True, future.result(timeout=timeout)
+#         except concurrent.futures.TimeoutError:
+#             print(f"⏱️ Timeout exceeded ({timeout} sec) — function killed.")
+#             return False, None
+
+def run_with_timeout(func, args=(), kwargs={}, timeout=30):
+    def wrapper(q, *args, **kwargs):
+        try:
+            result = func(*args, **kwargs)
+            q.put((True, result))
+        except Exception as e:
+            q.put((False, e))
+
+    q = multiprocessing.Queue()
+    p = multiprocessing.Process(target=wrapper, args=(q, *args), kwargs=kwargs)
+    p.start()
+    p.join(timeout)
+
+    if p.is_alive():
+        p.terminate()
+        p.join()
+        print(f"⏱️ Timeout exceeded ({timeout} sec) — function killed.")
+        return False, None
+
+    if not q.empty():
+        success, result = q.get()
+        if success:
+            return True, result
+        else:
+            raise result  # re-raise exception if needed
+
+    return False, None
+
+
+
+def time_it(func, *args, **kwargs):
+    """
+    Measure how long a function takes to run and return its result + time.
+    """
+    start = time.time()
+    result = func(*args, **kwargs)
+    end = time.time()
+    elapsed = end - start
+    print(f"⏱️ '{func.__name__}' took {elapsed:.3f} seconds")
+    return result, elapsed
+# --- Define Pricing Constants (for Gemini 1.5 Flash & text-embedding-004) ---    
+
+def unique_preserve_order(seq):
+    seen = set()
+    return [x for x in seq if not (x in seen or seen.add(x))]
+# Main execution
+def pipeline_with_gemini(accessions,stop_flag=None, niche_cases=None, save_df=None):
+  # output: country, sample_type, ethnic, location, money_cost, time_cost, explain
+  # there can be one accession number in the accessions
+  # Prices are per 1,000 tokens
+  # Before each big step:
+  if stop_flag is not None and stop_flag.value:
+    print(f"🛑 Stop detected before starting {accession}, aborting early...")
+    return {}
+  # PRICE_PER_1K_INPUT_LLM = 0.000075  # $0.075 per 1M tokens
+  # PRICE_PER_1K_OUTPUT_LLM = 0.0003   # $0.30 per 1M tokens
+  # PRICE_PER_1K_EMBEDDING_INPUT = 0.000025 # $0.025 per 1M tokens
+  # Gemini 2.5 Flash-Lite pricing per 1,000 tokens
+  PRICE_PER_1K_INPUT_LLM = 0.00010      # $0.10 per 1M input tokens
+  PRICE_PER_1K_OUTPUT_LLM = 0.00040     # $0.40 per 1M output tokens
+
+  # Embedding-001 pricing per 1,000 input tokens
+  PRICE_PER_1K_EMBEDDING_INPUT = 0.00015  # $0.15 per 1M input tokens  
+  if not accessions:
+    print("no input")
+    return None
+  else:  
+    accs_output = {}
+    #genai.configure(api_key=os.getenv("GOOGLE_API_KEY"))
+    genai.configure(api_key=os.getenv("GOOGLE_API_KEY_BACKUP"))  
+    for acc in accessions:
+      print("start gemini: ", acc)  
+      start = time.time()
+      total_cost_title = 0
+      jsonSM, links, article_text = {},[], ""
+      acc_score = { "isolate": "",
+                    "country":{},
+                   "sample_type":{},
+                   #"specific_location":{},
+                   #"ethnicity":{},
+                   "query_cost":total_cost_title,
+                   "time_cost":None,
+                   "source":links,
+                    "file_chunk":"",
+                   "file_all_output":""}
+      if niche_cases:
+        for niche in niche_cases:
+          acc_score[niche] = {}
+            
+      meta = mtdna_classifier.fetch_ncbi_metadata(acc)
+      country, spe_loc, ethnic, sample_type, col_date, iso, title, doi, pudID, features = meta["country"], meta["specific_location"], meta["ethnicity"], meta["sample_type"], meta["collection_date"], meta["isolate"], meta["title"], meta["doi"], meta["pubmed_id"], meta["all_features"]
+      acc_score["isolate"] = iso
+      print("meta: ",meta)  
+      meta_expand = smart_fallback.fetch_ncbi(acc)
+      print("meta expand: ", meta_expand)  
+      # set up step: create the folder to save document
+      chunk, all_output = "",""
+      if pudID: 
+        id = str(pudID)
+        saveTitle = title
+      else: 
+        try:
+          author_name = meta_expand["authors"].split(',')[0]  # Use last name only
+        except:
+          author_name = meta_expand["authors"] 
+        saveTitle = title + "_" + col_date + "_" + author_name
+        if title.lower() == "unknown" and col_date.lower()=="unknown" and   author_name.lower() == "unknown":
+            saveTitle += "_" + acc
+        id = "DirectSubmission"
+      # folder_path = Path("/content/drive/MyDrive/CollectData/MVP/mtDNA-Location-Classifier/data/"+str(id))
+      # if not folder_path.exists():
+      #     cmd = f'mkdir /content/drive/MyDrive/CollectData/MVP/mtDNA-Location-Classifier/data/{id}'
+      #     result = subprocess.run(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
+      #     print("data/"+str(id) +" created.")
+      # else:
+      #     print("data/"+str(id) +" already exists.")
+      # saveLinkFolder = "/content/drive/MyDrive/CollectData/MVP/mtDNA-Location-Classifier/data/"+str(id)
+      # parent_folder_id = get_or_create_drive_folder(GDRIVE_PARENT_FOLDER_NAME)
+      # data_folder_id = get_or_create_drive_folder(GDRIVE_DATA_FOLDER_NAME, parent_id=parent_folder_id)
+      # sample_folder_id = get_or_create_drive_folder(str(id), parent_id=data_folder_id)
+      data_folder_id = GDRIVE_DATA_FOLDER_NAME  # Use the shared folder directly
+      sample_folder_id = get_or_create_drive_folder(str(id), parent_id=data_folder_id)
+      print("sample folder id: ", sample_folder_id)
+      
+      # Define document names
+      if len(saveTitle) > 50:  
+          saveName = saveTitle[:50]
+          saveName = saveName.replace(" ", "_")
+          chunk_filename = f"{saveName}_merged_document.docx"
+          all_filename = f"{saveName}_all_merged_document.docx"
+      else:
+          saveName = saveTitle.replace(" ", "_")
+          chunk_filename = f"{saveName}_merged_document.docx"
+          all_filename = f"{saveName}_all_merged_document.docx"
+      print("chunk file name and all filename: ", chunk_filename, all_filename)  
+      # Define local temp paths for reading/writing
+      # import tempfile
+      # tmp_dir = tempfile.mkdtemp()
+      LOCAL_TEMP_DIR = "/mnt/data/generated_docs"
+      os.makedirs(LOCAL_TEMP_DIR, exist_ok=True)
+      file_chunk_path = os.path.join(LOCAL_TEMP_DIR, chunk_filename)
+      file_all_path = os.path.join(LOCAL_TEMP_DIR, all_filename)
+      # file_chunk_path = os.path.join(tempfile.gettempdir(), chunk_filename)
+      # file_all_path = os.path.join(tempfile.gettempdir(), all_filename)  
+      if stop_flag is not None and stop_flag.value:
+        print(f"🛑 Stop processing {accession}, aborting early...")
+        return {}
+      print("this is file chunk path: ", file_chunk_path)
+      chunk_id = find_drive_file(chunk_filename, sample_folder_id)
+      all_id = find_drive_file(all_filename, sample_folder_id)
+    
+      if chunk_id and all_id:
+        print("✅ Files already exist in Google Drive. Downloading them...")
+        chunk_exists = download_file_from_drive(chunk_filename, sample_folder_id, file_chunk_path)
+        all_exists = download_file_from_drive(all_filename, sample_folder_id, file_all_path)
+        acc_score["file_chunk"] = str(chunk_filename)
+        acc_score["file_all_output"] = str(all_filename)  
+        print("chunk_id and all_id: ")
+        print(chunk_id, all_id)  
+        print("file chunk and all output saved in acc score: ", acc_score["file_chunk"], acc_score["file_all_output"])  
+        file = drive_service.files().get(fileId="1LUJRTrq8yt4S4lLwCvTmlxaKqpr0nvEn", fields="id, name, parents, webViewLink").execute()
+        print("📄 Name:", file["name"])
+        print("📁 Parent folder ID:", file["parents"][0])
+        print("🔗 View link:", file["webViewLink"])
+  
+  
+        # Read and parse these into `chunk` and `all_output`
+      else:
+        # 🔥 Remove any stale local copies
+        if os.path.exists(file_chunk_path):
+            os.remove(file_chunk_path)
+            print(f"🗑️ Removed stale: {file_chunk_path}")
+        if os.path.exists(file_all_path):
+            os.remove(file_all_path)
+            print(f"🗑️ Removed stale: {file_all_path}")  
+      # 🔥 Remove the local file first if it exists
+      # if os.path.exists(file_chunk_path):
+      #   os.remove(file_chunk_path)
+      #   print("remove chunk path")  
+      # if os.path.exists(file_all_path):
+      #   os.remove(file_all_path)    
+      #   print("remove all path")  
+      # Try to download if already exists on Drive
+        chunk_exists = download_file_from_drive(chunk_filename, sample_folder_id, file_chunk_path)
+        all_exists = download_file_from_drive(all_filename, sample_folder_id, file_all_path)
+      print("chunk exist: ", chunk_exists)  
+      # first way: ncbi method
+      print("country.lower: ",country.lower())  
+      if country.lower() != "unknown":
+        stand_country = standardize_location.smart_country_lookup(country.lower())
+        print("stand_country: ", stand_country)  
+        if stand_country.lower() != "not found":
+          acc_score["country"][stand_country.lower()] = ["ncbi"]
+        else: acc_score["country"][country.lower()] = ["ncbi"]   
+      # if spe_loc.lower() != "unknown":
+      #   acc_score["specific_location"][spe_loc.lower()] = ["ncbi"]
+      # if ethnic.lower() != "unknown":
+      #   acc_score["ethnicity"][ethnic.lower()] = ["ncbi"]
+      if sample_type.lower() != "unknown":
+        acc_score["sample_type"][sample_type.lower()] = ["ncbi"]
+      # second way: LLM model
+      # Preprocess the input token
+      print(acc_score)  
+      accession, isolate = None, None
+      if acc != "unknown":  accession = acc
+      if iso != "unknown":  isolate = iso
+      if stop_flag is not None and stop_flag.value:
+        print(f"🛑 Stop processing {accession}, aborting early...")
+        return {}    
+      # check doi first
+      print("chunk filename: ", chunk_filename)    
+      if chunk_exists:
+        print("File chunk exists!")
+        if not chunk:
+            print("start to get chunk")
+            text, table, document_title = model.read_docx_text(file_chunk_path)
+            chunk = data_preprocess.normalize_for_overlap(text) + "\n" + data_preprocess.normalize_for_overlap(". ".join(table))
+        if str(chunk_filename) != "":
+            print("first time have chunk path at chunk exist: ", str(chunk_filename))
+            acc_score["file_chunk"] = str(chunk_filename)    
+      if all_exists:
+        print("File all output exists!")
+        if not all_output:
+            text_all, table_all, document_title_all = model.read_docx_text(file_all_path)
+            all_output = data_preprocess.normalize_for_overlap(text_all) + "\n" + data_preprocess.normalize_for_overlap(". ".join(table_all))
+        if str(all_filename) != "":
+            print("first time have all path at all exist: ", str(all_filename))
+            acc_score["file_all_output"] = str(all_filename)    
+      print("acc sscore for file all output and chunk: ", acc_score["file_all_output"], acc_score["file_chunk"])  
+      if len(acc_score["file_all_output"]) == 0 and len(acc_score["file_chunk"]) == 0:  
+          if doi != "unknown":
+            link = 'https://doi.org/' + doi
+            # get the file to create listOfFile for each id
+            print("link of doi: ", link)  
+            html = extractHTML.HTML("",link)
+            jsonSM = html.getSupMaterial()
+            article_text = html.getListSection()
+            if article_text:
+              if "Just a moment...Enable JavaScript and cookies to continue".lower() not in article_text.lower() or "403 Forbidden Request".lower() not in article_text.lower():
+                links.append(link)
+            if jsonSM:
+              links += sum((jsonSM[key] for key in jsonSM),[])
+          # no doi then google custom search api
+          if doi=="unknown" or len(article_text) == 0 or "Just a moment...Enable JavaScript and cookies to continue".lower() in article_text.lower() or "403 Forbidden Request".lower() in article_text.lower():
+            # might find the article
+            print("no article text, start tem link")  
+            #tem_links = mtdna_classifier.search_google_custom(title, 2)
+            tem_links = smart_fallback.smart_google_search(meta_expand)
+            print("tem links: ", tem_links)  
+            tem_link_acc = smart_fallback.google_accession_search(acc)
+            tem_links += tem_link_acc 
+            tem_links = unique_preserve_order(tem_links)
+            print("tem link before filtering: ", tem_links)  
+            # filter the quality link  
+            print("saveLinkFolder as sample folder id: ", sample_folder_id)  
+            print("start the smart filter link") 
+            if stop_flag is not None and stop_flag.value:
+                print(f"🛑 Stop processing {accession}, aborting early...")
+                return {}  
+            # success_process, output_process = run_with_timeout(smart_fallback.filter_links_by_metadata,args=(tem_links,sample_folder_id),kwargs={"accession":acc})
+            # if success_process:
+            #   links = output_process
+            #   print("yes succeed for smart filter link")
+            # else: 
+            #   print("no suceed, fallback to all tem links")
+            #   links = tem_links
+            links = smart_fallback.filter_links_by_metadata(tem_links, saveLinkFolder=sample_folder_id, accession=acc, stop_flag=stop_flag)
+          print("this is links: ",links)    
+          links = unique_preserve_order(links)
+          acc_score["source"] = links
+      else:
+        print("inside the try of reusing chunk or all output")  
+        #print("chunk filename: ", str(chunks_filename))  
+          
+        try:
+            temp_source = False
+            if save_df is not None and not save_df.empty:
+                print("save df not none")  
+                print("chunk file name: ",str(chunk_filename))
+                print("all filename: ",str(all_filename))
+                if acc_score["file_chunk"]:
+                  link = save_df.loc[save_df["file_chunk"]==acc_score["file_chunk"],"Sources"].iloc[0]
+                  #link = row["Sources"].iloc[0]
+                  if "http" in link:   
+                    print("yeah http in save df source") 
+                    acc_score["source"] = [x for x in link.split("\n") if x.strip()]#row["Sources"].tolist()
+                  else:  # temporary  
+                    print("tempo source") 
+                    #acc_score["source"] = [str(all_filename), str(chunks_filename)]
+                    temp_source = True
+                elif acc_score["file_all_output"]:
+                  link = save_df.loc[save_df["file_all_output"]==acc_score["file_all_output"],"Sources"].iloc[0]
+                  #link = row["Sources"].iloc[0]
+                  print(link)
+                  print("list of link")
+                  print([x for x in link.split("\n") if x.strip()])
+                  if "http" in link:    
+                    print("yeah http in save df source")
+                    acc_score["source"] = [x for x in link.split("\n") if x.strip()]#row["Sources"].tolist()   
+                  else:  # temporary  
+                    print("tempo source") 
+                    #acc_score["source"] = [str(all_filename), str(chunks_filename)]
+                    temp_source = True      
+                else:  # temporary  
+                  print("tempo source") 
+                  #acc_score["source"] = [str(file_all_path), str(file_chunk_path)]  
+                  temp_source = True
+            else:  # temporary  
+                print("tempo source") 
+                  #acc_score["source"] = [str(file_all_path), str(file_chunk_path)]  
+                temp_source = True      
+            if temp_source:
+                print("temp source is true so have to try again search link")
+                if doi != "unknown":
+                    link = 'https://doi.org/' + doi
+                    # get the file to create listOfFile for each id
+                    print("link of doi: ", link)  
+                    html = extractHTML.HTML("",link)
+                    jsonSM = html.getSupMaterial()
+                    article_text = html.getListSection()
+                    if article_text:
+                      if "Just a moment...Enable JavaScript and cookies to continue".lower() not in article_text.lower() or "403 Forbidden Request".lower() not in article_text.lower():
+                        links.append(link)
+                    if jsonSM:
+                      links += sum((jsonSM[key] for key in jsonSM),[])
+                  # no doi then google custom search api
+                if doi=="unknown" or len(article_text) == 0 or "Just a moment...Enable JavaScript and cookies to continue".lower() in article_text.lower() or "403 Forbidden Request".lower() in article_text.lower():
+                    # might find the article
+                    print("no article text, start tem link")  
+                    #tem_links = mtdna_classifier.search_google_custom(title, 2)
+                    tem_links = smart_fallback.smart_google_search(meta_expand)
+                    print("tem links: ", tem_links)  
+                    tem_link_acc = smart_fallback.google_accession_search(acc)
+                    tem_links += tem_link_acc 
+                    tem_links = unique_preserve_order(tem_links)
+                    print("tem link before filtering: ", tem_links)  
+                    # filter the quality link  
+                    print("saveLinkFolder as sample folder id: ", sample_folder_id)  
+                    print("start the smart filter link") 
+                    if stop_flag is not None and stop_flag.value:
+                        print(f"🛑 Stop processing {accession}, aborting early...")
+                        return {}  
+                    # success_process, output_process = run_with_timeout(smart_fallback.filter_links_by_metadata,args=(tem_links,sample_folder_id),kwargs={"accession":acc})
+                    # if success_process:
+                    #   links = output_process
+                    #   print("yes succeed for smart filter link")
+                    # else: 
+                    #   print("no suceed, fallback to all tem links")
+                    #   links = tem_links
+                    links = smart_fallback.filter_links_by_metadata(tem_links, saveLinkFolder=sample_folder_id, accession=acc, stop_flag=stop_flag)
+                print("this is links: ",links)    
+                links = unique_preserve_order(links)
+                acc_score["source"] = links
+        except:
+            print("except for source")  
+            acc_score["source"] = []
+      # chunk_path = "/"+saveTitle+"_merged_document.docx"
+      # all_path = "/"+saveTitle+"_all_merged_document.docx"
+      # # if chunk and all output not exist yet
+      # file_chunk_path = saveLinkFolder + chunk_path
+      # file_all_path = saveLinkFolder + all_path
+      # if os.path.exists(file_chunk_path):
+      #   print("File chunk exists!")
+      #   if not chunk:
+      #     text, table, document_title = model.read_docx_text(file_chunk_path)
+      #     chunk = data_preprocess.normalize_for_overlap(text) + "\n" + data_preprocess.normalize_for_overlap(". ".join(table))
+      # if os.path.exists(file_all_path):
+      #   print("File all output exists!")
+      #   if not all_output:
+      #     text_all, table_all, document_title_all = model.read_docx_text(file_all_path)
+      #     all_output = data_preprocess.normalize_for_overlap(text_all) + "\n" + data_preprocess.normalize_for_overlap(". ".join(table_all))
+      if stop_flag is not None and stop_flag.value:
+        print(f"🛑 Stop processing {accession}, aborting early...")
+        return {}
+      # print("chunk filename: ", chunk_filename)    
+      # if chunk_exists:
+      #   print("File chunk exists!")
+      #   if not chunk:
+      #       print("start to get chunk")
+      #       text, table, document_title = model.read_docx_text(file_chunk_path)
+      #       chunk = data_preprocess.normalize_for_overlap(text) + "\n" + data_preprocess.normalize_for_overlap(". ".join(table))
+      #   if str(chunk_filename) != "":
+      #       print("first time have chunk path at chunk exist: ", str(chunk_filename))
+      #       acc_score["file_chunk"] = str(chunk_filename)    
+      # if all_exists:
+      #   print("File all output exists!")
+      #   if not all_output:
+      #       text_all, table_all, document_title_all = model.read_docx_text(file_all_path)
+      #       all_output = data_preprocess.normalize_for_overlap(text_all) + "\n" + data_preprocess.normalize_for_overlap(". ".join(table_all))
+      #   if str(all_filename) != "":
+      #       print("first time have all path at all exist: ", str(all_filename))
+      #       acc_score["file_all_output"] = str(all_filename)    
+      if not chunk and not all_output:
+        print("not chunk and all output")  
+        # else: check if we can reuse these chunk and all output of existed accession to find another
+        if str(chunk_filename) != "":
+          print("first time have chunk path: ", str(chunk_filename))
+          acc_score["file_chunk"] = str(chunk_filename)
+        if str(all_filename) != "":
+          print("first time have all path: ", str(all_filename))
+          acc_score["file_all_output"] = str(all_filename)    
+        if links:
+          for link in links:
+              print(link)
+              # if len(all_output) > 1000*1000:
+              #   all_output = data_preprocess.normalize_for_overlap(all_output)
+              #   print("after normalizing all output: ", len(all_output))
+              if len(data_preprocess.normalize_for_overlap(all_output)) > 600000:
+                print("break here")
+                break
+              if iso != "unknown": query_kw = iso
+              else: query_kw = acc
+              #text_link, tables_link, final_input_link = data_preprocess.preprocess_document(link,saveLinkFolder, isolate=query_kw)
+              success_process, output_process = run_with_timeout(data_preprocess.preprocess_document,args=(link,sample_folder_id),kwargs={"isolate":query_kw,"accession":acc},timeout=100)
+              if stop_flag is not None and stop_flag.value:
+                print(f"🛑 Stop processing {accession}, aborting early...")
+                return {}
+              if success_process:
+                text_link, tables_link, final_input_link = output_process[0], output_process[1], output_process[2]
+                print("yes succeed for process document")
+              else: text_link, tables_link, final_input_link = "", "", ""  
+              context = data_preprocess.extract_context(final_input_link, query_kw)
+              if context !=  "Sample ID not found.":
+                if len(data_preprocess.normalize_for_overlap(chunk)) < 1000*1000:
+                  success_chunk, the_output_chunk = run_with_timeout(data_preprocess.merge_texts_skipping_overlap,args=(chunk, context))
+                  if stop_flag is not None and stop_flag.value:
+                    print(f"🛑 Stop processing {accession}, aborting early...")
+                    return {}
+                  if success_chunk:
+                    chunk = the_output_chunk#data_preprocess.merge_texts_skipping_overlap(all_output, final_input_link)
+                    print("yes succeed for chunk")
+                  else:
+                    chunk += context
+                    print("len context: ", len(context))
+                    print("basic fall back")
+                print("len chunk after: ", len(chunk))
+              if len(final_input_link) > 1000*1000:
+                if context !=  "Sample ID not found.":
+                  final_input_link =  context 
+                else:
+                  final_input_link = data_preprocess.normalize_for_overlap(final_input_link)
+                  if len(final_input_link) > 1000 *1000:
+                    final_input_link = final_input_link[:100000] 
+              if len(data_preprocess.normalize_for_overlap(all_output)) < int(100000) and len(final_input_link)<100000:
+                print("Running merge_texts_skipping_overlap with timeout")
+                success, the_output = run_with_timeout(data_preprocess.merge_texts_skipping_overlap,args=(all_output, final_input_link),timeout=30)
+                if stop_flag is not None and stop_flag.value:
+                  print(f"🛑 Stop processing {accession}, aborting early...")
+                  return {}
+                print("Returned from timeout logic")
+                if success:
+                  all_output = the_output#data_preprocess.merge_texts_skipping_overlap(all_output, final_input_link)
+                  print("yes succeed")
+                else:
+                  print("len all output: ", len(all_output))
+                  print("len final input link: ", len(final_input_link))
+                  all_output += final_input_link
+                  print("len final input: ", len(final_input_link))
+                  print("basic fall back")
+              else:
+                  print("both/either all output or final link too large more than 100000")
+                  print("len all output: ", len(all_output))
+                  print("len final input link: ", len(final_input_link))
+                  all_output += final_input_link
+                  print("len final input: ", len(final_input_link))
+                  print("basic fall back")    
+              print("len all output after: ", len(all_output))
+          #country_pro, chunk, all_output = data_preprocess.process_inputToken(links, saveLinkFolder, accession=accession, isolate=isolate)
+        if stop_flag is not None and stop_flag.value:
+          print(f"🛑 Stop processing {accession}, aborting early...")
+          return {}  
+        else:
+          chunk = "Collection_date: " + col_date +". Isolate: " + iso + ". Title: " + title + ". Features: " + features
+          all_output = "Collection_date: " + col_date +". Isolate: " + iso + ". Title: " + title + ". Features: " + features
+        if not chunk: chunk = "Collection_date: " + col_date +". Isolate: " + iso + ". Title: " + title + ". Features: " + features
+        if not all_output:  all_output = "Collection_date: " + col_date +". Isolate: " + iso + ". Title: " + title + ". Features: " + features
+        if len(all_output) > 1*1024*1024: 
+          all_output = data_preprocess.normalize_for_overlap(all_output)
+          if len(all_output) > 1*1024*1024:
+            all_output = all_output[:1*1024*1024]
+        print("chunk len: ", len(chunk))
+        print("all output len: ", len(all_output))    
+        data_preprocess.save_text_to_docx(chunk, file_chunk_path)
+        data_preprocess.save_text_to_docx(all_output, file_all_path)
+        # Later when saving new files
+        # data_preprocess.save_text_to_docx(chunk, chunk_filename, sample_folder_id)
+        # data_preprocess.save_text_to_docx(all_output, all_filename, sample_folder_id)
+        
+        # Upload to Drive
+        result_chunk_upload = upload_file_to_drive(file_chunk_path, chunk_filename, sample_folder_id)
+        result_all_upload = upload_file_to_drive(file_all_path, all_filename, sample_folder_id)
+        print("UPLOAD RESULT FOR CHUNK: ", result_chunk_upload)
+        print(f"🔗 Uploaded file: https://drive.google.com/file/d/{result_chunk_upload}/view")  
+        print("here 1")
+          
+      # else:
+      #   final_input = ""
+      #   if all_output:
+      #     final_input = all_output
+      #   else:  
+      #     if chunk: final_input = chunk
+      #   #data_preprocess.merge_texts_skipping_overlap(final_input, all_output)
+      #   if final_input:
+      #     keywords = []
+      #     if iso != "unknown":  keywords.append(iso)
+      #     if acc != "unknown":  keywords.append(acc)
+      #     for keyword in keywords:
+      #       chunkBFS = data_preprocess.get_contextual_sentences_BFS(final_input, keyword)
+      #       countryDFS, chunkDFS = data_preprocess.get_contextual_sentences_DFS(final_input, keyword)
+      #       chunk = data_preprocess.merge_texts_skipping_overlap(chunk, chunkDFS)
+      #       chunk = data_preprocess.merge_texts_skipping_overlap(chunk, chunkBFS)
+          
+      # Define paths for cached RAG assets
+      # faiss_index_path = saveLinkFolder+"/faiss_index.bin"
+      # document_chunks_path = saveLinkFolder+"/document_chunks.json"
+      # structured_lookup_path = saveLinkFolder+"/structured_lookup.json"
+      print("here 2")
+      faiss_filename = "faiss_index.bin"
+      chunks_filename = "document_chunks.json"
+      lookup_filename = "structured_lookup.json"
+      print("name of faiss: ", faiss_filename)  
+          
+      faiss_index_path = os.path.join(LOCAL_TEMP_DIR, faiss_filename)
+      document_chunks_path = os.path.join(LOCAL_TEMP_DIR, chunks_filename)
+      structured_lookup_path = os.path.join(LOCAL_TEMP_DIR, lookup_filename)
+      print("name if faiss path: ", faiss_index_path)  
+      # 🔥 Remove the local file first if it exists
+      print("start faiss id and also the sample folder id is: ", sample_folder_id)  
+      faiss_id = find_drive_file(faiss_filename, sample_folder_id)
+      print("done faiss id")  
+      document_id = find_drive_file(chunks_filename, sample_folder_id)
+      structure_id = find_drive_file(lookup_filename, sample_folder_id)  
+      if faiss_id and document_id and structure_id:
+        print("✅ 3 Files already exist in Google Drive. Downloading them...")
+        download_file_from_drive(faiss_filename, sample_folder_id, faiss_index_path)
+        download_file_from_drive(chunks_filename, sample_folder_id, document_chunks_path)
+        download_file_from_drive(lookup_filename, sample_folder_id, structured_lookup_path)  
+        # Read and parse these into `chunk` and `all_output`
+      else:
+        "one of id not exist"  
+        if os.path.exists(faiss_index_path):
+            print("faiss index exist and start to remove: ", faiss_index_path)
+            os.remove(faiss_index_path)
+        if os.path.exists(document_chunks_path):
+            os.remove(document_chunks_path)
+        if os.path.exists(structured_lookup_path):
+            os.remove(structured_lookup_path)    
+        print("start to download the faiss, chunk, lookup")
+            
+        download_file_from_drive(faiss_filename, sample_folder_id, faiss_index_path)
+        download_file_from_drive(chunks_filename, sample_folder_id, document_chunks_path)
+        download_file_from_drive(lookup_filename, sample_folder_id, structured_lookup_path)
+      try:
+          print("try gemini 2.5")
+          print("move to load rag")
+          master_structured_lookup, faiss_index, document_chunks = model.load_rag_assets(
+              faiss_index_path, document_chunks_path, structured_lookup_path
+          )
+    
+          global_llm_model_for_counting_tokens = genai.GenerativeModel('gemini-1.5-flash-latest')
+          if not all_output:
+            if chunk: all_output = chunk
+            else: all_output = "Collection_date: " + col_date +". Isolate: " + iso + ". Title: " + title + ". Features: " + features
+          if faiss_index is None:
+              print("\nBuilding RAG assets (structured lookup, FAISS index, chunks)...")
+              total_doc_embedding_tokens = global_llm_model_for_counting_tokens.count_tokens(
+                  all_output
+              ).total_tokens
+    
+              initial_embedding_cost = (total_doc_embedding_tokens / 1000) * PRICE_PER_1K_EMBEDDING_INPUT
+              total_cost_title += initial_embedding_cost
+              print(f"Initial one-time embedding cost for '{file_all_path}' ({total_doc_embedding_tokens} tokens): ${initial_embedding_cost:.6f}")
+    
+    
+              master_structured_lookup, faiss_index, document_chunks, plain_text_content = model.build_vector_index_and_data(
+                  file_all_path, faiss_index_path, document_chunks_path, structured_lookup_path
+              )
+          else:
+              print("\nRAG assets loaded from file. No re-embedding of entire document will occur.")
+              plain_text_content_all, table_strings_all, document_title_all = model.read_docx_text(file_all_path)
+              master_structured_lookup['document_title'] = master_structured_lookup.get('document_title', document_title_all)
+          if stop_flag is not None and stop_flag.value:
+            print(f"🛑 Stop processing {accession}, aborting early...")
+            return {}  
+          primary_word = iso
+          alternative_word = acc
+          print(f"\n--- General Query: Primary='{primary_word}' (Alternative='{alternative_word}') ---")
+          if features.lower() not in all_output.lower():  
+            all_output += ". NCBI Features: " + features
+          # country, sample_type, method_used, ethnic, spe_loc, total_query_cost =  model.query_document_info(
+          #     primary_word, alternative_word, meta, master_structured_lookup, faiss_index, document_chunks, 
+          #     model.call_llm_api, chunk=chunk, all_output=all_output)
+          print("this is chunk for the model")
+          print(chunk)
+          print("this is all output for the model")
+          print(all_output)  
+          if stop_flag is not None and stop_flag.value:
+            print(f"🛑 Stop processing {accession}, aborting early...")
+            return {}
+          country, sample_type, method_used, country_explanation, sample_type_explanation, total_query_cost =  model.query_document_info(
+              primary_word, alternative_word, meta, master_structured_lookup, faiss_index, document_chunks, 
+              model.call_llm_api, chunk=chunk, all_output=all_output)
+          print("pass query of 2.5")
+      except:
+          print("try gemini 1.5")
+          country, sample_type, ethnic, spe_loc, method_used, country_explanation, sample_type_explanation, ethnicity_explanation, specific_loc_explanation, total_query_cost = model.query_document_info(
+            primary_word, alternative_word, meta, master_structured_lookup, faiss_index, document_chunks, 
+            model.call_llm_api, chunk=chunk, all_output=all_output, model_ai="gemini-1.5-flash-latest")      
+          print("yeah pass the query of 1.5")
+      print("country using ai: ", country)
+      print("sample type using ai: ", sample_type)  
+      # if len(country) == 0: country = "unknown"
+      # if len(sample_type) == 0: sample_type = "unknown"    
+      # if country_explanation: country_explanation = "-"+country_explanation        
+      # else: country_explanation = ""
+      # if sample_type_explanation: sample_type_explanation = "-"+sample_type_explanation
+      # else: sample_type_explanation = ""
+      if len(country) == 0: country = "unknown"
+      if len(sample_type) == 0: sample_type = "unknown"    
+      if country_explanation and country_explanation!="unknown": country_explanation = "-"+country_explanation        
+      else: country_explanation = ""
+      if sample_type_explanation and sample_type_explanation!="unknown": sample_type_explanation = "-"+sample_type_explanation
+      else: sample_type_explanation = ""
+          
+      if method_used == "unknown": method_used = ""
+      if country.lower() != "unknown":
+        stand_country = standardize_location.smart_country_lookup(country.lower())
+        if stand_country.lower() != "not found":
+          if stand_country.lower() in acc_score["country"]:
+            if country_explanation:
+              acc_score["country"][stand_country.lower()].append(method_used + country_explanation)
+          else:
+            acc_score["country"][stand_country.lower()] = [method_used + country_explanation]
+        else:
+          if country.lower() in acc_score["country"]:
+            if country_explanation:
+              if len(method_used + country_explanation) > 0:
+                acc_score["country"][country.lower()].append(method_used + country_explanation)
+          else:
+            if len(method_used + country_explanation) > 0:
+              acc_score["country"][country.lower()] = [method_used + country_explanation]
+      # if spe_loc.lower() != "unknown":
+      #   if spe_loc.lower() in acc_score["specific_location"]:
+      #     acc_score["specific_location"][spe_loc.lower()].append(method_used)
+      #   else:
+      #     acc_score["specific_location"][spe_loc.lower()] = [method_used]
+      # if ethnic.lower() != "unknown":
+      #   if ethnic.lower() in acc_score["ethnicity"]:
+      #     acc_score["ethnicity"][ethnic.lower()].append(method_used)
+      #   else:
+      #     acc_score["ethnicity"][ethnic.lower()] = [method_used]
+      if sample_type.lower() != "unknown":
+        if sample_type.lower() in acc_score["sample_type"]:
+          if len(method_used + sample_type_explanation) > 0:
+            acc_score["sample_type"][sample_type.lower()].append(method_used + sample_type_explanation)
+        else:
+          if len(method_used + sample_type_explanation)> 0:
+            acc_score["sample_type"][sample_type.lower()] = [method_used + sample_type_explanation]
+      total_cost_title += total_query_cost
+      if stop_flag is not None and stop_flag.value:
+        print(f"🛑 Stop processing {accession}, aborting early...")
+        return {}
+      # last resort: combine all information to give all output otherwise unknown
+      if len(acc_score["country"]) == 0 or len(acc_score["sample_type"]) == 0 or acc_score["country"] == "unknown" or acc_score["sample_type"] == "unknown":   
+        text = ""
+        for key in meta_expand:
+          text += str(key) + ": " + meta_expand[key] + "\n"    
+        if len(data_preprocess.normalize_for_overlap(all_output)) > 0:
+          text += data_preprocess.normalize_for_overlap(all_output)          
+        if len(data_preprocess.normalize_for_overlap(chunk)) > 0:
+          text += data_preprocess.normalize_for_overlap(chunk)            
+        text += ". NCBI Features: " + features 
+        print("this is text for the last resort model")
+        print(text)  
+        country, sample_type, method_used, country_explanation, sample_type_explanation, total_query_cost =  model.query_document_info(
+            primary_word, alternative_word, meta, master_structured_lookup, faiss_index, document_chunks, 
+            model.call_llm_api, chunk=text, all_output=text)  
+        print("this is last resort results: ")
+        print("country: ", country)
+        print("sample type: ", sample_type)  
+        if len(country) == 0: country = "unknown"
+        if len(sample_type) == 0: sample_type = "unknown"    
+        # if country_explanation: country_explanation = "-"+country_explanation        
+        # else: country_explanation = ""
+        # if sample_type_explanation: sample_type_explanation = "-"+sample_type_explanation
+        # else: sample_type_explanation = ""
+        if country_explanation and country_explanation!="unknown": country_explanation = "-"+country_explanation        
+        else: country_explanation = ""
+        if sample_type_explanation and sample_type_explanation!="unknown": sample_type_explanation = "-"+sample_type_explanation
+        else: sample_type_explanation = ""
+            
+        if method_used == "unknown": method_used = ""
+        if country.lower() != "unknown":
+          stand_country = standardize_location.smart_country_lookup(country.lower())
+          if stand_country.lower() != "not found":
+            if stand_country.lower() in acc_score["country"]:
+              if country_explanation:
+                acc_score["country"][stand_country.lower()].append(method_used + country_explanation)
+            else:
+              acc_score["country"][stand_country.lower()] = [method_used + country_explanation]
+          else:
+            if country.lower() in acc_score["country"]:
+              if country_explanation:
+                if len(method_used + country_explanation) > 0:
+                  acc_score["country"][country.lower()].append(method_used + country_explanation)
+            else:
+              if len(method_used + country_explanation) > 0:
+                acc_score["country"][country.lower()] = [method_used + country_explanation]
+        if sample_type.lower() != "unknown":
+            if sample_type.lower() in acc_score["sample_type"]:
+              if len(method_used + sample_type_explanation) > 0:
+                acc_score["sample_type"][sample_type.lower()].append(method_used + sample_type_explanation)
+            else:
+              if len(method_used + sample_type_explanation)> 0:
+                acc_score["sample_type"][sample_type.lower()] = [method_used + sample_type_explanation]          
+        total_cost_title += total_query_cost
+      end = time.time()
+      #total_cost_title += total_query_cost
+      acc_score["query_cost"] = f"{total_cost_title:.6f}"
+      elapsed = end - start
+      acc_score["time_cost"] = f"{elapsed:.3f} seconds"
+      accs_output[acc] = acc_score
+      print(accs_output[acc])
+      
+    return accs_output

core/smart_fallback.py

@@ -0,0 +1,259 @@
+from Bio import Entrez, Medline
+#import model
+import core.mtdna_classifier
+from core.NER.html import extractHTML
+import core.data_preprocess
+import core.pipeline
+# Setup
+def fetch_ncbi(accession_number):
+  try:
+    Entrez.email = "your.email@example.com" # Required by NCBI, REPLACE WITH YOUR EMAIL
+    handle = Entrez.efetch(db="nucleotide", id=str(accession_number), rettype="gb", retmode="xml")
+    record = Entrez.read(handle)
+    handle.close()
+    outputs = {"authors":"unknown",
+              "institution":"unknown",
+              "isolate":"unknown",
+              "definition":"unknown",
+              "title":"unknown",
+              "seq_comment":"unknown",
+              "collection_date":"unknown" } #'GBSeq_update-date': '25-OCT-2023', 'GBSeq_create-date' 
+    gb_seq = None
+    # Validate record structure: It should be a list with at least one element (a dict)
+    if isinstance(record, list) and len(record) > 0:
+        if isinstance(record[0], dict):
+            gb_seq = record[0]
+        else:
+            print(f"Warning: record[0] is not a dictionary for {accession_number}. Type: {type(record[0])}")
+        # extract collection date  
+        if "GBSeq_create-date" in gb_seq and outputs["collection_date"]=="unknown":
+          outputs["collection_date"] = gb_seq["GBSeq_create-date"]
+        else:
+          if "GBSeq_update-date" in gb_seq and outputs["collection_date"]=="unknown":
+            outputs["collection_date"] = gb_seq["GBSeq_update-date"]
+        # extract definition
+        if "GBSeq_definition" in gb_seq and outputs["definition"]=="unknown":
+          outputs["definition"] = gb_seq["GBSeq_definition"]
+        # extract related-reference things
+        if "GBSeq_references" in gb_seq:
+          for ref in gb_seq["GBSeq_references"]:
+            # extract authors
+            if "GBReference_authors" in ref and outputs["authors"]=="unknown":
+              outputs["authors"] = "and ".join(ref["GBReference_authors"])
+            # extract title
+            if "GBReference_title" in ref and outputs["title"]=="unknown":
+              outputs["title"] = ref["GBReference_title"]  
+            #  extract submitted journal
+            if 'GBReference_journal' in ref and outputs["institution"]=="unknown":
+              outputs["institution"] = ref['GBReference_journal']
+        # extract seq_comment
+        if 'GBSeq_comment'in gb_seq and outputs["seq_comment"]=="unknown":
+          outputs["seq_comment"] = gb_seq["GBSeq_comment"]
+        # extract isolate
+        if "GBSeq_feature-table" in gb_seq:
+          if 'GBFeature_quals' in gb_seq["GBSeq_feature-table"][0]:
+            for ref in gb_seq["GBSeq_feature-table"][0]["GBFeature_quals"]:
+              if ref['GBQualifier_name'] == "isolate" and outputs["isolate"]=="unknown":
+                outputs["isolate"] = ref["GBQualifier_value"]
+    else:
+        print(f"Warning: No valid record or empty record list from NCBI for {accession_number}.")
+
+    # If gb_seq is still None, return defaults
+    if gb_seq is None:
+        return {"authors":"unknown",
+              "institution":"unknown",
+              "isolate":"unknown",
+              "definition":"unknown",
+              "title":"unknown",
+              "seq_comment":"unknown",
+              "collection_date":"unknown" }
+    return outputs   
+  except:
+    print("error in fetching ncbi data")   
+    return {"authors":"unknown",
+              "institution":"unknown",
+              "isolate":"unknown",
+              "definition":"unknown",
+              "title":"unknown",
+              "seq_comment":"unknown",
+              "collection_date":"unknown" }
+# Fallback if NCBI crashed or cannot find accession on NBCI
+def google_accession_search(accession_id):
+    """
+    Search for metadata by accession ID using Google Custom Search.
+    Falls back to known biological databases and archives.
+    """
+    queries = [
+        f"{accession_id}",
+        f"{accession_id} site:ncbi.nlm.nih.gov",
+        f"{accession_id} site:pubmed.ncbi.nlm.nih.gov",
+        f"{accession_id} site:europepmc.org",
+        f"{accession_id} site:researchgate.net",
+        f"{accession_id} mtDNA",
+        f"{accession_id} mitochondrial DNA"
+    ]
+    
+    links = []
+    for query in queries:
+        search_results = mtdna_classifier.search_google_custom(query, 2)
+        for link in search_results:
+            if link not in links:
+                links.append(link)
+    return links
+             
+# Method 1: Smarter Google
+def smart_google_queries(metadata: dict):
+    queries = []
+
+    # Extract useful fields
+    isolate = metadata.get("isolate")
+    author = metadata.get("authors")
+    institution = metadata.get("institution")
+    title = metadata.get("title")
+    combined = []
+    # Construct queries
+    if isolate and isolate!="unknown" and isolate!="Unpublished":
+        queries.append(f'"{isolate}" mitochondrial DNA')
+        queries.append(f'"{isolate}" site:ncbi.nlm.nih.gov')
+        
+    if author and author!="unknown" and author!="Unpublished":
+        # try:
+        #   author_name = ".".join(author.split(' ')[0].split(".")[:-1])  # Use last name only
+        # except:
+        #   try:
+        #     author_name = author.split(',')[0]  # Use last name only
+        #   except:  
+        #     author_name = author
+        try:
+            author_name = author.split(',')[0]  # Use last name only
+        except:  
+            author_name = author
+        queries.append(f'"{author_name}" mitochondrial DNA')
+        queries.append(f'"{author_name}" mtDNA site:researchgate.net')
+        
+    if institution and institution!="unknown" and institution!="Unpublished":
+        try:
+          short_inst = ",".join(institution.split(',')[:2])  # Take first part of institution
+        except:
+          try:
+            short_inst = institution.split(',')[0]
+          except:
+            short_inst = institution
+        queries.append(f'"{short_inst}" mtDNA sequence')
+        #queries.append(f'"{short_inst}" isolate site:nature.com')
+    if title and title!='unknown' and title!="Unpublished":
+      if title!="Direct Submission":
+        queries.append(title)  
+          
+    return queries
+
+def filter_links_by_metadata(search_results, saveLinkFolder, accession=None, stop_flag=None):
+    TRUSTED_DOMAINS = [
+    "ncbi.nlm.nih.gov",
+    "pubmed.ncbi.nlm.nih.gov",
+    "pmc.ncbi.nlm.nih.gov",
+    "biorxiv.org",
+    "researchgate.net",
+    "nature.com",
+    "sciencedirect.com"
+    ]
+    if stop_flag is not None and stop_flag.value:
+        print(f"🛑 Stop detected {accession}, aborting early...")
+        return []
+    def is_trusted_link(link):
+      for domain in TRUSTED_DOMAINS:
+        if domain in link:
+          return True
+      return False
+    def is_relevant_title_snippet(link, saveLinkFolder, accession=None):
+      output = []
+      keywords = ["mtDNA", "mitochondrial", "accession", "isolate", "Homo sapiens", "sequence"]
+      if accession:
+        keywords = [accession] + keywords
+      title_snippet = link.lower()
+      print("save link folder inside this filter function: ", saveLinkFolder)  
+      success_process, output_process = pipeline.run_with_timeout(data_preprocess.extract_text,args=(link,saveLinkFolder),timeout=60)
+      if stop_flag is not None and stop_flag.value:
+        print(f"🛑 Stop detected {accession}, aborting early...")
+        return []
+      if success_process:
+          article_text = output_process
+          print("yes succeed for getting article text")
+      else: 
+          print("no suceed, fallback to no link")
+          article_text = ""  
+      #article_text = data_preprocess.extract_text(link,saveLinkFolder)
+      print("article text")
+      #print(article_text)  
+      if stop_flag is not None and stop_flag.value:
+        print(f"🛑 Stop detected {accession}, aborting early...")
+        return []  
+      try:
+        ext = link.split(".")[-1].lower()
+        if ext not in ["pdf", "docx", "xlsx"]:
+            html = extractHTML.HTML("", link)
+            if stop_flag is not None and stop_flag.value:
+                print(f"🛑 Stop detected {accession}, aborting early...")
+                return []
+            jsonSM = html.getSupMaterial()
+            if jsonSM:
+                output += sum((jsonSM[key] for key in jsonSM), [])
+      except Exception:
+        pass  # continue silently
+      for keyword in keywords:
+        if keyword.lower() in article_text.lower():
+          if link not in output:
+            output.append([link,keyword.lower()])
+          print("link and keyword for article text: ", link, keyword)    
+          return output
+        if keyword.lower() in title_snippet.lower():
+          if link not in output:
+            output.append([link,keyword.lower()])
+          print("link and keyword for title: ", link, keyword)    
+          return output
+      return output
+    
+    filtered = []
+    better_filter = []
+    if len(search_results) > 0:
+      for link in search_results:
+          # if is_trusted_link(link):
+          #   if link not in filtered:
+          #     filtered.append(link)
+          # else:
+          print(link)
+          if stop_flag is not None and stop_flag.value:
+            print(f"🛑 Stop detected {accession}, aborting early...")
+            return []
+          if link:    
+            output_link = is_relevant_title_snippet(link,saveLinkFolder, accession)
+            print("output link: ")
+            print(output_link)
+            for out_link in output_link:
+              if isinstance(out_link,list) and len(out_link) > 1:
+                print(out_link)
+                kw = out_link[1]
+                print("kw and acc: ", kw, accession.lower())  
+                if accession and kw == accession.lower():
+                  better_filter.append(out_link[0])
+                filtered.append(out_link[0])
+              else: filtered.append(out_link)
+          print("done with link and here is filter: ",filtered)      
+    if better_filter:
+      filtered = better_filter      
+    return filtered
+
+def smart_google_search(metadata):
+  queries = smart_google_queries(metadata)
+  links = []
+  for q in queries:
+      #print("\n🔍 Query:", q)
+      results = mtdna_classifier.search_google_custom(q,2)
+      for link in results:
+          #print(f"- {link}")
+          if link not in links:
+              links.append(link)
+  #filter_links = filter_links_by_metadata(links)
+  return links
+# Method 2: Prompt LLM better or better ai search api with all
+# the total information from even ncbi and all search

core/standardize_location.py

@@ -0,0 +1,90 @@
+import re, os
+import requests
+import core.model
+# Normalize input
+def normalize_key(text):
+    return re.sub(r"[^a-z0-9]", "", text.strip().lower())
+
+# Search for city/place (normal flow)
+def get_country_from_geonames(city_name):
+    url = os.environ["URL_SEARCHJSON"]  
+    username = os.environ["USERNAME_GEO"]
+    print("geoname: ", cityname)
+    params = {
+        "q": city_name,
+        "maxRows": 1,
+        "username": username
+    }
+    try:
+        r = requests.get(url, params=params, timeout=5)
+        data = r.json()
+        if data.get("geonames"):
+            return data["geonames"][0]["countryName"]
+    except Exception as e:
+        print("GeoNames searchJSON error:", e)
+    return None
+
+# Search for country info using alpha-2/3 codes or name
+def get_country_from_countryinfo(input_code):
+    url = os.environ["URL_COUNTRYJSON"] 
+    username = os.environ["USERNAME_GEO"]
+    print("countryINFO: ", input_code)
+    params = {
+        "username": username
+    }
+    try:
+        r = requests.get(url, params=params, timeout=5)
+        data = r.json()
+        if data.get("geonames"):
+            input_code = input_code.strip().upper()
+            for country in data["geonames"]:
+                # Match against country name, country code (alpha-2), iso alpha-3
+                if input_code in [
+                    country.get("countryName", "").upper(),
+                    country.get("countryCode", "").upper(),
+                    country.get("isoAlpha3", "").upper()
+                ]:
+                    return country["countryName"]
+    except Exception as e:
+        print("GeoNames countryInfoJSON error:", e)
+    return None
+
+# Combined smart lookup
+def smart_country_lookup(user_input):
+    try: 
+        raw_input = user_input.strip()
+        normalized = re.sub(r"[^a-zA-Z0-9]", "", user_input).upper()  # normalize for codes (no strip spaces!)
+        print("raw input for smart country lookup: ",raw_input, ". Normalized country: ", normalized)
+        # Special case: if user writes "UK: London" → split and take main country part
+        if ":" in raw_input:
+            raw_input = raw_input.split(":")[0].strip()  # only take "UK"
+        # First try as country code (if 2-3 letters or common abbreviation)
+        if len(normalized) <= 3:
+          if normalized.upper() in ["UK","U.K","U.K."]:
+            country = get_country_from_geonames(normalized.upper())
+            print("get_country_from_geonames(normalized.upper()) ", country)  
+            if country:
+              return country
+          else:  
+            country = get_country_from_countryinfo(raw_input)
+            print("get_country_from_countryinfo(raw_input) ", country)  
+            if country:
+                return country
+        print(raw_input)        
+        country = get_country_from_countryinfo(raw_input)  # try full names
+        print("get_country_from_countryinfo(raw_input) ", country)
+        if country:
+            return country
+        # Otherwise, treat as city/place
+        country = get_country_from_geonames(raw_input)
+        print("get_country_from_geonames(raw_input) ", country)
+        if country:
+            return country
+    
+        return "Not found"
+    except:
+        country = model.get_country_from_text(user_input)
+        if country.lower() !="unknown":
+            return country
+        else:
+            return "Not found"

core/upgradeClassify.py

@@ -0,0 +1,276 @@
+import re
+import spacy
+from nltk.tokenize import sent_tokenize, word_tokenize
+import nltk
+nltk.download('punkt_tab')
+#import coreferee
+import copy
+from sentence_transformers import SentenceTransformer, util
+from sklearn.cluster import DBSCAN
+from sklearn.metrics.pairwise import cosine_distances
+from collections import defaultdict
+import numpy as np
+#from mtdna_classifier import infer_fromQAModel
+# 1. SENTENCE-BERT MODEL
+# Step 1: Preprocess the text
+def normalize_text(text):
+    # Normalize various separators to "-"
+    text = re.sub(r'\s*(–+|—+|--+>|–>|->|-->|to|→|➝|➔|➡)\s*', '-', text, flags=re.IGNORECASE)
+    # Fix GEN10GEN30 → GEN10-GEN30
+    text = re.sub(r'\b([a-zA-Z]+)(\d+)(\1)(\d+)\b', r'\1\2-\1\4', text)
+    # Fix GEN10-30 → GEN10-GEN30
+    text = re.sub(r'\b([a-zA-Z]+)(\d+)-(\d+)\b', r'\1\2-\1\3', text)
+    return text
+
+def preprocess_text(text):
+    normalized = normalize_text(text)
+    sentences = sent_tokenize(normalized)
+    return [re.sub(r"[^a-zA-Z0-9\s\-]", "", s).strip() for s in sentences]
+
+# Before step 2, check NLP cache to avoid calling it muliple times:
+# Global model cache
+_spacy_models = {}
+
+def get_spacy_model(model_name, add_coreferee=False):
+    global _spacy_models
+    if model_name not in _spacy_models:
+        nlp = spacy.load(model_name)
+        if add_coreferee and "coreferee" not in nlp.pipe_names:
+            nlp.add_pipe("coreferee")
+        _spacy_models[model_name] = nlp
+    return _spacy_models[model_name]
+
+# Step 2: NER to Extract Locations and Sample Names
+def extract_entities(text, sample_id=None):
+    nlp = get_spacy_model("en_core_web_sm")
+    doc = nlp(text)
+    
+    # Filter entities by GPE, but exclude things that match sample ID format
+    gpe_candidates = [ent.text for ent in doc.ents if ent.label_ == "GPE"]
+    
+    # Remove entries that match SAMPLE ID patterns like XXX123 or similar
+    gpe_filtered = [gpe for gpe in gpe_candidates if not re.fullmatch(r'[A-Z]{2,5}\d{2,4}', gpe.strip())]
+    
+    # Optional: further filter known invalid patterns (e.g., things shorter than 3 chars, numeric only)
+    gpe_filtered = [gpe for gpe in gpe_filtered if len(gpe) > 2 and not gpe.strip().isdigit()]
+    
+    if sample_id is None:
+        return list(set(gpe_filtered)), []
+    else:
+        sample_prefix = re.match(r'[A-Z]+', sample_id).group()
+        samples = re.findall(rf'{sample_prefix}\d+', text)
+        return list(set(gpe_filtered)), list(set(samples))
+
+# Step 3: Build a Soft Matching Layer
+# Handle patterns like "BRU1–BRU20" and identify BRU18 as part of it.
+def is_sample_in_range(sample_id, sentence):
+    # Match prefix up to digits
+    sample_prefix_match = re.match(r'^([A-Z0-9]+?)(?=\d+$)', sample_id)
+    sample_number_match = re.search(r'(\d+)$', sample_id)
+
+    if not sample_prefix_match or not sample_number_match:
+        return False
+
+    sample_prefix = sample_prefix_match.group(1)
+    sample_number = int(sample_number_match.group(1))
+    sentence = normalize_text(sentence)
+    # Case 1: Full prefix on both sides
+    pattern1 = rf'{sample_prefix}(\d+)\s*-\s*{sample_prefix}(\d+)'
+    for match in re.findall(pattern1, sentence):
+        start, end = int(match[0]), int(match[1])
+        if start <= sample_number <= end:
+            return True
+
+    # Case 2: Prefix only on first number
+    pattern2 = rf'{sample_prefix}(\d+)\s*-\s*(\d+)'
+    for match in re.findall(pattern2, sentence):
+        start, end = int(match[0]), int(match[1])
+        if start <= sample_number <= end:
+            return True
+
+    return False
+
+# Step 4: Use coreferree to merge the sentences have same coreference # still cannot cause packages conflict
+# ========== HEURISTIC GROUP → LOCATION MAPPERS ==========
+# === Generalized version to replace your old extract_sample_to_group_general ===
+# === Generalized version to replace your old extract_group_to_location_general ===
+def extract_population_locations(text):
+    text = normalize_text(text)
+    pattern = r'([A-Za-z ,\-]+)\n([A-Z]+\d*)\n([A-Za-z ,\-]+)\n([A-Za-z ,\-]+)'
+    pop_to_location = {}
+
+    for match in re.finditer(pattern, text, flags=re.IGNORECASE):
+        _, pop_code, region, country = match.groups()
+        pop_to_location[pop_code.upper()] = f"{region.strip()}\n{country.strip()}"
+
+    return pop_to_location
+ 
+def extract_sample_ranges(text):
+    text = normalize_text(text)
+    # Updated pattern to handle punctuation and line breaks
+    pattern = r'\b([A-Z0-9]+\d+)[–\-]([A-Z0-9]+\d+)[,:\.\s]*([A-Z0-9]+\d+)\b'
+    sample_to_pop = {}
+    for match in re.finditer(pattern, text, flags=re.IGNORECASE):
+        start_id, end_id, pop_code = match.groups()
+        start_prefix = re.match(r'^([A-Z0-9]+?)(?=\d+$)', start_id, re.IGNORECASE).group(1).upper()
+        end_prefix = re.match(r'^([A-Z0-9]+?)(?=\d+$)', end_id, re.IGNORECASE).group(1).upper()
+        if start_prefix != end_prefix:
+            continue
+        start_num = int(re.search(r'(\d+)$', start_id).group())
+        end_num = int(re.search(r'(\d+)$', end_id).group())
+        for i in range(start_num, end_num + 1):
+            sample_id = f"{start_prefix}{i:03d}"
+            sample_to_pop[sample_id] = pop_code.upper()
+
+    return sample_to_pop
+
+def filter_context_for_sample(sample_id, full_text, window_size=2):
+
+    # Normalize and tokenize
+    full_text = normalize_text(full_text)
+    sentences = sent_tokenize(full_text)
+
+    # Step 1: Find indices with direct mention or range match
+    match_indices = [
+        i for i, s in enumerate(sentences)
+        if sample_id in s or is_sample_in_range(sample_id, s)
+    ]
+
+    # Step 2: Get sample → group mapping from full text
+    sample_to_group = extract_sample_ranges(full_text)
+    group_id = sample_to_group.get(sample_id)
+
+    # Step 3: Find group-related sentences
+    group_indices = []
+    if group_id:
+        for i, s in enumerate(sentences):
+            if group_id in s:
+                group_indices.append(i)
+
+    # Step 4: Collect sentences within window
+    selected_indices = set()
+    if len(match_indices + group_indices) > 0:
+      for i in match_indices + group_indices:
+          start = max(0, i - window_size)
+          end = min(len(sentences), i + window_size + 1)
+          selected_indices.update(range(start, end))
+
+      filtered_sentences = [sentences[i] for i in sorted(selected_indices)]
+      return " ".join(filtered_sentences) 
+    return full_text  
+# Load the SpaCy transformer model with coreferee
+def mergeCorefSen(text):
+  sen = preprocess_text(text)
+  return sen
+
+# Before step 5 and below, let check transformer cache to avoid calling again
+# Global SBERT model cache
+_sbert_models = {}
+
+def get_sbert_model(model_name="all-MiniLM-L6-v2"):
+    global _sbert_models
+    if model_name not in _sbert_models:
+        _sbert_models[model_name] = SentenceTransformer(model_name)
+    return _sbert_models[model_name]
+
+# Step 5: Sentence-BERT retriever → Find top paragraphs related to keyword.
+'''Use sentence transformers to embed the sentence that mentions the sample and
+compare it to sentences that mention locations.'''
+
+def find_top_para(sample_id, text,top_k=5):
+    sentences = mergeCorefSen(text)
+    model = get_sbert_model("all-mpnet-base-v2")
+    embeddings = model.encode(sentences, convert_to_tensor=True)
+
+    # Find the sentence that best matches the sample_id
+    sample_matches = [s for s in sentences if sample_id in s or is_sample_in_range(sample_id, s)]
+    if not sample_matches:
+        return [],"No context found for sample"
+
+    sample_embedding = model.encode(sample_matches[0], convert_to_tensor=True)
+    cos_scores = util.pytorch_cos_sim(sample_embedding, embeddings)[0]
+
+    # Get top-k most similar sentence indices
+    top_indices = cos_scores.argsort(descending=True)[:top_k]
+    return top_indices, sentences
+
+# Step 6: DBSCAN to cluster the group of similar paragraphs.
+def clusterPara(tokens):
+  # Load Sentence-BERT model
+  sbert_model = get_sbert_model("all-mpnet-base-v2")
+  sentence_embeddings = sbert_model.encode(tokens)
+
+  # Compute cosine distance matrix
+  distance_matrix = cosine_distances(sentence_embeddings)
+
+  # DBSCAN clustering
+  clustering_model = DBSCAN(eps=0.3, min_samples=1, metric="precomputed")
+  cluster_labels = clustering_model.fit_predict(distance_matrix)
+
+  # Group sentences by cluster
+  clusters = defaultdict(list)
+  cluster_embeddings = defaultdict(list)
+  sentence_to_cluster = {}
+  for i, label in enumerate(cluster_labels):
+    clusters[label].append(tokens[i])
+    cluster_embeddings[label].append(sentence_embeddings[i])
+    sentence_to_cluster[tokens[i]] = label
+  # Compute cluster centroids
+  centroids = {
+      label: np.mean(embs, axis=0)
+      for label, embs in cluster_embeddings.items()
+  }
+  return clusters, sentence_to_cluster, centroids
+
+def rankSenFromCluster(clusters, sentence_to_cluster, centroids, target_sentence):
+  target_cluster = sentence_to_cluster[target_sentence]
+  target_centroid = centroids[target_cluster]
+  sen_rank = []
+  sen_order = list(sentence_to_cluster.keys())
+  # Compute distances to other cluster centroids
+  dists = []
+  for label, centroid in centroids.items():
+    dist = cosine_distances([target_centroid], [centroid])[0][0]
+    dists.append((label, dist))
+  dists.sort(key=lambda x: x[1])  # sort by proximity
+  for d in dists:
+    cluster = clusters[d[0]]
+    for sen in cluster:
+      if sen != target_sentence:
+        sen_rank.append(sen_order.index(sen))
+  return sen_rank
+# Step 7: Final Inference Wrapper
+def infer_location_for_sample(sample_id, context_text):
+    # Go through each of the top sentences in order
+    top_indices, sentences = find_top_para(sample_id, context_text,top_k=5)
+    if top_indices==[] or sentences == "No context found for sample":
+      return "No clear location found in top matches"
+    clusters, sentence_to_cluster, centroids = clusterPara(sentences)
+    topRankSen_DBSCAN = []
+    mostTopSen = ""
+    locations = ""
+    i = 0
+    while len(locations) == 0 or i < len(top_indices):
+      # Firstly, start with the top-ranked Sentence-BERT result
+      idx = top_indices[i]
+      best_sentence = sentences[idx]
+      if i == 0:
+        mostTopSen = best_sentence
+      locations, _ = extract_entities(best_sentence, sample_id)
+      if locations:
+        return locations
+      # If no location, then look for sample overlap in the same DBSCAN cluster
+      # Compute distances to other cluster centroids
+      if len(topRankSen_DBSCAN)==0 and mostTopSen:
+        topRankSen_DBSCAN = rankSenFromCluster(clusters, sentence_to_cluster, centroids, mostTopSen)
+      if i >= len(topRankSen_DBSCAN): break
+      idx_DBSCAN = topRankSen_DBSCAN[i]
+      best_sentence_DBSCAN = sentences[idx_DBSCAN]
+      locations, _ = extract_entities(best_sentence, sample_id)
+      if locations:
+        return locations
+      # If no, then backtrack to next best Sentence-BERT sentence (such as 2nd rank sentence), and repeat step 1 and 2 until run out
+      i += 1
+    # Last resort: LLM (e.g. chatGPT, deepseek, etc.)
+    #if len(locations) == 0:
+    return "No clear location found in top matches"

env.yaml

@@ -0,0 +1,8 @@
+name: mtDNA
+channels:
+  - conda-forge
+dependencies:
+  - python=3.10
+  - pip
+  - pip:
+    - -r requirements.txt