Spaces:
Runtime error
Runtime error
Update chunk_python_code.py
Browse files- chunk_python_code.py +6 -6
chunk_python_code.py
CHANGED
|
@@ -51,7 +51,7 @@ def _iterate_ast(python_code, documents, file_path):
|
|
| 51 |
|
| 52 |
if isinstance(first_level_node, ast.ClassDef):
|
| 53 |
documents.extend(
|
| 54 |
-
|
| 55 |
elif isinstance(first_level_node, ast.FunctionDef):
|
| 56 |
documents.extend(
|
| 57 |
_chunk_first_level_func_node(first_level_node, documents, python_code))
|
|
@@ -60,7 +60,7 @@ def _iterate_ast(python_code, documents, file_path):
|
|
| 60 |
_chunk_first_level_assign_node(first_level_node, documents, python_code))
|
| 61 |
else:
|
| 62 |
documents.extend(
|
| 63 |
-
|
| 64 |
|
| 65 |
|
| 66 |
def _chunk_import_only_python_code(python_code, file_path):
|
|
@@ -84,18 +84,18 @@ def _chunk_import_only_python_code(python_code, file_path):
|
|
| 84 |
documents.append(doc)
|
| 85 |
return documents
|
| 86 |
|
| 87 |
-
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 88 |
|
| 89 |
|
| 90 |
-
def
|
| 91 |
documents = []
|
| 92 |
documents.extend(
|
| 93 |
chunk_python_code_by_character)
|
| 94 |
|
| 95 |
return documents
|
| 96 |
|
|
|
|
| 97 |
|
| 98 |
-
def
|
| 99 |
documents = []
|
| 100 |
text_splitter = RecursiveCharacterTextSplitter(
|
| 101 |
chunk_size=512,
|
|
@@ -156,7 +156,7 @@ def _chunk_first_level_assign_node(ast_node, python_code):
|
|
| 156 |
|
| 157 |
return documents
|
| 158 |
|
| 159 |
-
def
|
| 160 |
"""
|
| 161 |
Handles classes at the first level of the AST.
|
| 162 |
"""
|
|
|
|
| 51 |
|
| 52 |
if isinstance(first_level_node, ast.ClassDef):
|
| 53 |
documents.extend(
|
| 54 |
+
_handle_first_level_class(first_level_node, documents, python_code))
|
| 55 |
elif isinstance(first_level_node, ast.FunctionDef):
|
| 56 |
documents.extend(
|
| 57 |
_chunk_first_level_func_node(first_level_node, documents, python_code))
|
|
|
|
| 60 |
_chunk_first_level_assign_node(first_level_node, documents, python_code))
|
| 61 |
else:
|
| 62 |
documents.extend(
|
| 63 |
+
_handle_notdefined_case(python_code))
|
| 64 |
|
| 65 |
|
| 66 |
def _chunk_import_only_python_code(python_code, file_path):
|
|
|
|
| 84 |
documents.append(doc)
|
| 85 |
return documents
|
| 86 |
|
|
|
|
| 87 |
|
| 88 |
|
| 89 |
+
def _handle_notdefined_case(python_code):
|
| 90 |
documents = []
|
| 91 |
documents.extend(
|
| 92 |
chunk_python_code_by_character)
|
| 93 |
|
| 94 |
return documents
|
| 95 |
|
| 96 |
+
from langchain.text_splitter import RecursiveCharacterTextSplitter
|
| 97 |
|
| 98 |
+
def _chunk_python_code_by_character(python_code):
|
| 99 |
documents = []
|
| 100 |
text_splitter = RecursiveCharacterTextSplitter(
|
| 101 |
chunk_size=512,
|
|
|
|
| 156 |
|
| 157 |
return documents
|
| 158 |
|
| 159 |
+
def _handle_first_level_class(ast_node , python_code):
|
| 160 |
"""
|
| 161 |
Handles classes at the first level of the AST.
|
| 162 |
"""
|