Spaces:
Runtime error
Runtime error
Update app.py
Browse files
app.py
CHANGED
|
@@ -52,6 +52,19 @@ api = HfApi()
|
|
| 52 |
def split_python_code_into_chunks(texts, file_paths):
|
| 53 |
chunks = []
|
| 54 |
for text, file_path in zip(texts, file_paths):
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
document_chunks = chunk_python_code_with_metadata(text, file_path)
|
| 56 |
chunks.extend(document_chunks)
|
| 57 |
return chunks
|
|
|
|
| 52 |
def split_python_code_into_chunks(texts, file_paths):
|
| 53 |
chunks = []
|
| 54 |
for text, file_path in zip(texts, file_paths):
|
| 55 |
+
"""
|
| 56 |
+
Custom made python code splitter, algorithm iterates through child nodes of ast-tree(max child depth = 2)
|
| 57 |
+
aims to have full body of methods along signature (+ can handle decorators) in a chunk and adds method specific metadata
|
| 58 |
+
e.g visbility: public, _internal
|
| 59 |
+
type: "class", "methods", "command"(CLI commands)
|
| 60 |
+
source:
|
| 61 |
+
|
| 62 |
+
|
| 63 |
+
with the intend to use a filter when retrieving potentaion useful snippets.
|
| 64 |
+
|
| 65 |
+
|
| 66 |
+
|
| 67 |
+
"""
|
| 68 |
document_chunks = chunk_python_code_with_metadata(text, file_path)
|
| 69 |
chunks.extend(document_chunks)
|
| 70 |
return chunks
|