Update chunk_python_code.py

chunk_python_code.py

@@ -1,9 +1,9 @@
 import ast
 from langchain.schema import Document 
 
-def split_python_code_into_chunks(texts, file_paths):
+def chunk_pythoncode_and_add_metadata(code_files_content, code_files_source):
     chunks = [] 
-    for text, file_path in zip(texts, file_paths):
+    for code_file_content, code_file_source in zip(code_files_content, code_files_source):
         """
         Custom made python code splitter, algorithm iterates through child nodes of ast-tree(max child depth = 2)
         aims to have full body of methods along signature (+ can handle decorators) in a chunk and adds method specific metadata
@@ -17,13 +17,13 @@ def split_python_code_into_chunks(texts, file_paths):
 
         
         """
-        document_chunks = chunk_python_code_with_metadata(text, file_path)
+        document_chunks = generate_code_chunks_with_metadata(code_files_content, code_files_source)
         chunks.extend(document_chunks)   
     return chunks
 
 
 # Split text into chunks
-def split_into_chunks(texts, references, chunk_size, chunk_overlap):
+def chunk_text_and_add_metadata(texts, references, chunk_size, chunk_overlap):
     text_splitter = RecursiveCharacterTextSplitter(chunk_size=chunk_size, chunk_overlap=chunk_overlap)
     chunks = []
 
@@ -41,7 +41,7 @@ def split_into_chunks(texts, references, chunk_size, chunk_overlap):
     return chunks
 
 
-def chunk_python_code_with_metadata(python_code, file_path):
+def generate_code_chunks_with_metadata(code_file_content, code_file_source):
     """
     Custom Python Code Splitter
     chunks python file by length of func/method body
@@ -55,7 +55,7 @@ def chunk_python_code_with_metadata(python_code, file_path):
     documents = []
    #print(f"Processing file: {file_path}")
   
-    _iterate_ast(python_code, documents, file_path)
+    _iterate_ast(code_file_content, documents, file_path)
     # Determine usage based on the file_path
     if file_path.startswith("kadi_apy/lib/"):
         usage = "kadi-apy python library"
@@ -72,45 +72,45 @@ def chunk_python_code_with_metadata(python_code, file_path):
     return documents
 
 
-def _iterate_ast(python_code, documents, file_path):
+def _iterate_ast(code_file_content, documents, file_path):
     """
     Parses the AST of the given Python file and delegates
     handling to specific methods based on node types.
     """
-    tree = ast.parse(python_code, filename=file_path)
+    tree = ast.parse(code_file_content, filename=file_path)
 
     first_level_nodes = list(ast.iter_child_nodes(tree))
 
     # Check if there are no first-level nodes
     if not first_level_nodes:
         documents.extend(
-            _chunk_nodeless_python_code(python_code, file_path))
+            _chunk_nodeless_code_file_content(code_file_content, file_path))
         return
 
     all_imports = all(isinstance(node, (ast.Import, ast.ImportFrom)) for node in first_level_nodes)
     if all_imports:
         documents.extend(
-            _chunk_import_only_python_code(python_code, file_path))
+            _chunk_import_only_code_file_content(code_file_content, file_path))
     
     # Iterate over first-level nodes
     for first_level_node in ast.iter_child_nodes(tree):
         
         if isinstance(first_level_node, ast.ClassDef):
             documents.extend(
-                _handle_first_level_class(first_level_node, python_code))
+                _handle_first_level_class(first_level_node, code_file_content))
         elif isinstance(first_level_node, ast.FunctionDef):
             documents.extend(
-                _chunk_first_level_func_node(first_level_node, python_code))
+                _chunk_first_level_func_node(first_level_node, code_file_content))
         elif isinstance(first_level_node, ast.Assign):
             documents.extend(
-                _chunk_first_level_assign_node(first_level_node, python_code))
+                _chunk_first_level_assign_node(first_level_node, code_file_content))
      #   else: 
      #       documents.extend(
-     #           _handle_not_defined_case(python_code))
+     #           _handle_not_defined_case(code_file_content))
              
 
 
-def _handle_first_level_class(ast_node , python_code):
+def _handle_first_level_class(ast_node , code_file_content):
     """
     Handles classes at the first level of the AST.
     """
@@ -118,7 +118,7 @@ def _handle_first_level_class(ast_node , python_code):
     class_start_line = ast_node.lineno
     class_body_lines = [child.lineno for child in ast_node.body if isinstance(child, ast.FunctionDef)]
     class_end_line = min(class_body_lines, default=ast_node.end_lineno) - 1
-    class_source = '\n'.join(python_code.splitlines()[class_start_line-1:class_end_line])
+    class_source = '\n'.join(code_file_content.splitlines()[class_start_line-1:class_end_line])
 
     metadata = {
         "type": "class",
@@ -141,7 +141,7 @@ def _handle_first_level_class(ast_node , python_code):
                 if second_level_node.decorator_list else second_level_node.lineno
             )
             method_end_line = second_level_node.end_lineno
-            method_source = '\n'.join(python_code.splitlines()[method_start_line-1:method_end_line])
+            method_source = '\n'.join(code_file_content.splitlines()[method_start_line-1:method_end_line])
 
             visibility = "internal" if second_level_node.name.startswith("_") else "public"
 
@@ -159,14 +159,14 @@ def _handle_first_level_class(ast_node , python_code):
     return documents
 
 
-def _handle_not_defined_case(python_code):
+def _handle_not_defined_case(code_file_content):
     documents = []
     documents.extend(
-        _chunk_python_code_by_character(python_code))
+        _chunk_code_file_content_by_character(code_file_content))
     return documents
     
 
-def _chunk_first_level_func_node(ast_node, python_code):
+def _chunk_first_level_func_node(ast_node, code_file_content):
     """
     Handles functions at the first level of the AST.
     """
@@ -176,7 +176,7 @@ def _chunk_first_level_func_node(ast_node, python_code):
         if ast_node.decorator_list else ast_node.lineno
     )
     function_end_line = ast_node.end_lineno
-    function_source = '\n'.join(python_code.splitlines()[function_start_line-1:function_end_line])
+    function_source = '\n'.join(code_file_content.splitlines()[function_start_line-1:function_end_line])
 
     visibility = "internal" if ast_node.name.startswith("_") else "public"
 
@@ -205,7 +205,7 @@ def _chunk_first_level_func_node(ast_node, python_code):
 
 
 
-def _chunk_first_level_assign_node(ast_node, python_code):
+def _chunk_first_level_assign_node(ast_node, code_file_content):
     
     """
     Handles assignment statements at the first level of the AST.
@@ -213,7 +213,7 @@ def _chunk_first_level_assign_node(ast_node, python_code):
     documents = []
     assign_start_line = ast_node.lineno
     assign_end_line = ast_node.end_lineno
-    assign_source = '\n'.join(python_code.splitlines()[assign_start_line-1:assign_end_line])
+    assign_source = '\n'.join(code_file_content.splitlines()[assign_start_line-1:assign_end_line])
 
     # Create metadata without imports
     metadata = {"type": "Assign"}
@@ -229,7 +229,7 @@ def _chunk_first_level_assign_node(ast_node, python_code):
 
 
     
-def _chunk_import_only_python_code(python_code, file_path):
+def _chunk_import_only_code_file_content(code_file_content, file_path):
     """
     Handles cases where the first-level nodes are only imports.
     """
@@ -244,13 +244,13 @@ def _chunk_import_only_python_code(python_code, file_path):
 
     # Create and store a Document with the full source code
     doc = Document(
-        page_content=python_code,
+        page_content=code_file_content,
         metadata=metadata
     )
     documents.append(doc)
     return documents
 
-def _chunk_nodeless_python_code(python_code, file_path):
+def _chunk_nodeless_code_file_content(code_file_content, file_path):
     """
     Handles cases where no top-level nodes are found in the AST.
     """
@@ -265,7 +265,7 @@ def _chunk_nodeless_python_code(python_code, file_path):
 
     # Create and store a Document with the full source code
     doc = Document(
-        page_content=python_code,
+        page_content=code_file_content,
         metadata=metadata
     )
     documents.append(doc)
@@ -277,7 +277,7 @@ def _chunk_nodeless_python_code(python_code, file_path):
 from langchain.text_splitter import RecursiveCharacterTextSplitter
 
 
-def _chunk_python_code_by_character(python_code):
+def _chunk_code_file_content_by_character(code_file_content):
     documents = []
     text_splitter = RecursiveCharacterTextSplitter(
         chunk_size=512,         
@@ -285,7 +285,7 @@ def _chunk_python_code_by_character(python_code):
         separators=[]
     )
     
-    chunks = text_splitter.split_text(python_code)
+    chunks = text_splitter.split_text(code_file_content)
     
     for chunk in chunks:
         doc = Document(