Update chunk_python_code.py

chunk_python_code.py

@@ -1,6 +1,44 @@
 import ast
 from langchain.schema import Document 
 
+def split_python_code_into_chunks(texts, file_paths):
+    chunks = [] 
+    for text, file_path in zip(texts, file_paths):
+        """
+        Custom made python code splitter, algorithm iterates through child nodes of ast-tree(max child depth = 2)
+        aims to have full body of methods along signature (+ can handle decorators) in a chunk and adds method specific metadata
+        e.g visbility: public, _internal
+            type: "class", "methods", "command"(CLI commands)
+            source: 
+        
+        
+        with the intend to use a filter when retrieving potentaion useful snippets. 
+        
+
+        
+        """
+        document_chunks = chunk_python_code_with_metadata(text, file_path)
+        chunks.extend(document_chunks)   
+    return chunks
+
+
+# Split text into chunks
+def split_into_chunks(texts, references, chunk_size, chunk_overlap):
+    text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
+    chunks = []
+
+    for text, reference in zip(texts, references):
+        chunks.extend([
+            Document(
+                page_content=chunk,
+                metadata={
+                    "source": reference,
+                    "usage": "doc"
+                }
+            ) 
+            for chunk in text_splitter.split_text(text)
+        ])
+    return chunks
 
 
 def chunk_python_code_with_metadata(python_code, file_path):