Update chunking

run.py

@@ -2,7 +2,7 @@
 # Title:  Gradio Interface to LLM-chatbot with dynamic RAG-funcionality and ChromaDB
 # Author: Andreas Fischer
 # Date:   October 10th, 2024
-# Last update: October 15th, 2024
+# Last update: October 22th, 2024
 ##########################################################################################
 
 import os
@@ -82,8 +82,8 @@ def format_prompt0(message, history):
 
 def format_prompt(message, history, system=None, RAGAddon=None, system2=None, zeichenlimit=None,historylimit=4, removeHTML=False):
   if zeichenlimit is None: zeichenlimit=1000000000 # :-)
-  startOfString="<s>"  #<s> [INST] U1 [/INST] A1</s> [INST] U2 [/INST] A2</s>
-  template0=" [INST] {system} [/INST]</s>"  
+  startOfString="<s>" 
+  template0=" [INST] {system} [/INST] </s>"  #" [INST] {system} [/INST] </s>"  vs " [INST]{system}\n  [/INST] </s>"
   template1=" [INST] {message} [/INST]"
   template2=" {response}</s>"
   prompt = ""
@@ -155,13 +155,58 @@ def convertPDF(pdf_file, allow_ocr=False):
 # Function for splitting text with overlap
 #------------------------------------------
 
-def split_with_overlap(text,chunk_size=3500, overlap=700):
- chunks=[]
- step=max(1,chunk_size-overlap)
- for i in range(0,len(text),step):
-   end=min(i+chunk_size,len(text))
-   chunks.append(text[i:end])
- return chunks
+def split_with_overlap0(text,chunk_size=3500, overlap=700):
+  """ Split text in chunks based on number of characters (chunk_size) with chunks overlapping (overlap)"""
+  chunks=[]
+  step=max(1,chunk_size-overlap)
+  for i in range(0,len(text),step):
+    end=min(i+chunk_size,len(text))
+    chunks.append(text[i:end])
+  return chunks
+
+import re
+def split_with_overlap(text, chunk_size=3500, overlap=700, pattern=r'([.!;?][ \n\r]|[\n\r]{2,})', variant=1, verbose=False):
+    """ Split text in chunks based on regex (pattern) matches. By default the pattern is '([.!;?][ \\n\\r]|[\\n\\r]{2,})' Chunks are no longer than a certain number of characters (chunk_size) with chunks overlapping (overlap).
+     By default (variant=1) chunking is based on complete sentences, but it's also possible to split only within the left overlap region and within the rest of the chunk-size (variant==2) or strictly within both overlap-regions (variant=3).    
+    """
+    chunks = []
+    overlap=min(overlap,chunk_size) # Overlap kann nicht größer sein als chunk_size
+    step = max(1, chunk_size - overlap) # step richtet sich nach chunk_size und overlap
+    def find_pattern(text): # Funktion zur Suche nach dem Muster
+        return re.search(pattern, text)
+    i, lastEnd = 0,0
+    while i<len(text):
+      print("i="+str(i))
+      end = min(i + chunk_size, len(text)) 
+      pattern_match = find_pattern(text[i:end]) # erstes Vorkommnis (if any)
+      matchesStart = [x.start() for x in re.finditer(pattern, text[i:end])] # start aller matches
+      matchesEnd =   [x.start() for x in re.finditer(pattern, text[i:end])] # end aller matches
+      step = max(1, chunk_size - overlap) # Normalerweise beträgt ein Step chunk_size - overlap     
+      if pattern_match: # Wenn (mindestens) ein Satzzeichen gefunden wurde        
+        for s in matchesStart: # gehe jedes Satzzeichen durch
+          if ((variant<=2 and s>=overlap) or (variant==3 and s>=overlap and s>(chunk_size-overlap))): # wenn das Satzzeichen nicht im Overlap links liegt (1) oder zusätzlich im reechten Overlap liegt (2) - wobei letzteres unvollständige Sätze bedeuten kann
+            end=s+i+1 # Setze end auf den Start des Patterns/Satzzeichens im gesamten Text
+            if(verbose==True): print("***move end:"+str(end)+"; step="+str(step))
+            if(s<(chunk_size-overlap)):step=min(step,max(1,s-overlap)) # Springe mit step höchstens zum Ende des Satzzeichens (nur erforderlich, wenn end nicht im Overlap)
+        if ((variant==1 and i>0) or (variant>=2 and pattern_match.start()<overlap and i>0)): # wenn das erste Satzzeichen im Overlap liegt
+          i=i+pattern_match.start()+1     # Verzichte auf Textteile vor dem ersten Satzzeichen                               
+      if(verbose==True): print("i="+str(i)+"; end="+str(end)+"; step="+str(step)+"; len="+str(len(text))+"; match="+str(pattern_match)+"; text="+text[i:end]+"; rest="+text[end:])
+      if(end>lastEnd): # wenn das Ende sich verschoben hat (und nicht nur den Satzbeginn zu einem bereits bekannten Satz abschneidet)
+        chunks.append(text[i:end])        
+      lastEnd=end
+      if(verbose==True): print("Text at position "+str(i)+": "+text[i:end])
+      i += step
+    if(len(text[end:])>0): chunks.append(text[end:]) # Ergänze am ende etwaigen Rest 
+    return chunks
+
+fiveChars=  "(?<![ \n\(]bspw|[ \n]inkl)"
+fourChars=  "(?<![ \n\(]sog|[ \n]Mio|[ \n]Mrd|[ \n]Tsd|[ \n]Tel)" 
+threeChars= "(?<!www|bzw|etc|ggf|[ \n\(]al|[ \n\(]St|[ \n\(]dh|[ \n\(]va|[ \n\(]ca|[ \n\(]Dr|[ \n\(]Hr|[ \n\(]Fr|[0-9]ff)"
+twoChars=   "(?<![ \n\(][A-Za-zΆ-Ωά-ωäöüß])"
+oneChars=   "(?<![0-9.])"
+sentenceRegex="(?<=[^.]{4})"+fiveChars+fourChars+threeChars+twoChars+oneChars+"[.?!](?![A-Za-zΆ-Ωά-ωäöüß0-9.!?'\"])"
+sectionRegex="\n[ ]*\n[\n ]*"
+splitRegex="("+sentenceRegex+"|"+sectionRegex+")"
 
 
 #---------------------------------------------------------------
@@ -169,16 +214,22 @@ def split_with_overlap(text,chunk_size=3500, overlap=700):
 #---------------------------------------------------------------
 
 def add_doc(path, session):
+  global device
   print("def add_doc!")
   print(path)
   anhang=False
   if(str.lower(path).endswith(".pdf") and os.path.exists(path)):
       doc=convertPDF(path)
-      if(len(doc[0])>5): 
-        gr.Info("PDF uploaded to DB_"+str(session)+", start Indexing excerpt (first 5 pages)!")        
+      if(len(doc[0])>5):         
+        if(not "cuda" in device):
+          doc="\n\n".join(doc[0][0:5]) 
+          gr.Info("PDF uploaded to DB_"+str(session)+", start Indexing excerpt (first 5 pages on CPU setups)!")
+        else:
+          doc="\n\n".join(doc[0]) 
+          gr.Info("PDF uploaded to DB_"+str(session)+", start Indexing!")                
       else:
-        gr.Info("PDF uploaded to DB_"+str(session)+", start Indexing!")            
-      doc="\n\n".join(doc[0][0:5]) 
+        doc="\n\n".join(doc[0]) 
+        gr.Info("PDF uploaded to DB_"+str(session)+", start Indexing!")                 
       anhang=True
   else:
     gr.Info("No PDF attached - answer based on DB_"+str(session)+".")          
@@ -196,29 +247,28 @@ def add_doc(path, session):
     collection = client.get_collection(
       name=dbName, embedding_function=embeddingModel)
   if(anhang==True):
-    corpus=split_with_overlap(doc,3500,700) 
-    print(len(corpus))
+    corpus=split_with_overlap(doc,3500,700,pattern=splitRegex) 
+    print("Length of corpus: "+str(len(corpus)))
+    print("Corpus:"+str(corpus))
     then = datetime.now()
     x=collection.get(include=[])["ids"]
     print(len(x))
     if(len(x)==0):
       chunkSize=40000
-      for i in range(round(len(corpus)/chunkSize+0.5)): 
+      for i in range(round(len(corpus)/chunkSize+0.5)): #0 is first batch, 3 is last (incomplete) batch given 133497 texts
         print("embed batch "+str(i)+" of "+str(round(len(corpus)/chunkSize+0.5)))
         ids=list(range(i*chunkSize,(i*chunkSize+chunkSize)))
         batch=corpus[i*chunkSize:(i*chunkSize+chunkSize)]
         textIDs=[str(id) for id in ids[0:len(batch)]]
         ids=[str(id+len(x)+1) for id in ids[0:len(batch)]] # id refers to chromadb-unique ID      
         collection.add(documents=batch, ids=ids, 
-          metadatas=[{"date": str("2024-10-10")} for b in batch]) 
+          metadatas=[{"date": str("2024-10-10")} for b in batch]) #"textID":textIDs, "id":ids, 
         print("finished batch "+str(i)+" of "+str(round(len(corpus)/40000+0.5)))  
     now = datetime.now()
     gr.Info(f"Indexing complete!")
-    print(now-then) 
+    print(now-then) #zu viel GB für sentences (GPU), bzw. 0:00:10.375087 für chunks
   return(collection)
 
-#split_with_overlap("test me if you can",2,1)
-
 
 #--------------------------------------------------------
 # Function for response to user queries and pot. addenda
@@ -249,7 +299,7 @@ def multimodal_response(message, history, dropdown, hfToken, request: gr.Request
   print(str(client.list_collections()))
   x=collection.get(include=[])["ids"]  
   context=collection.query(query_texts=[query], n_results=1)
-  context=["<context "+str(i)+"> "+str(c)+"</context "+str(i)+">" for i,c in enumerate(context["documents"][0])]
+  context=["<Kontext "+str(i)+"> "+str(c)+"</Kontext "+str(i)+">" for i,c in enumerate(context["documents"][0])]
   gr.Info("Kontext:\n"+str(context))    
   generate_kwargs = dict(
         temperature=float(0.9),
@@ -305,3 +355,4 @@ i=gr.ChatInterface(multimodal_response,
   ])
 i.launch() #allowed_paths=["."])
 
+