Delete splitBioModels.py
Browse files- splitBioModels.py +0 -50
splitBioModels.py
DELETED
|
@@ -1,50 +0,0 @@
|
|
| 1 |
-
from langchain_text_splitters import CharacterTextSplitter
|
| 2 |
-
import os
|
| 3 |
-
from typing import List, Optional
|
| 4 |
-
|
| 5 |
-
def splitBioModels(directory: str, final_items: Optional[List[str]] = None) -> List[str]:
|
| 6 |
-
"""Separates BioModel database based on indentation
|
| 7 |
-
|
| 8 |
-
Args:
|
| 9 |
-
directory (str): Relative path to the folder containing the files.
|
| 10 |
-
final_items (Optional[List[str]]): A list to store the split content. If None, a new list will be created.
|
| 11 |
-
|
| 12 |
-
Returns:
|
| 13 |
-
List[str]: A list of text chunks split from the BioModel files.
|
| 14 |
-
"""
|
| 15 |
-
text_splitter2 = CharacterTextSplitter(
|
| 16 |
-
separator=" // ",
|
| 17 |
-
chunk_size=1000000000,
|
| 18 |
-
chunk_overlap=20,
|
| 19 |
-
length_function=len,
|
| 20 |
-
is_separator_regex=False
|
| 21 |
-
)
|
| 22 |
-
|
| 23 |
-
if final_items is None:
|
| 24 |
-
final_items = []
|
| 25 |
-
final_items = list(final_items)
|
| 26 |
-
|
| 27 |
-
directory_path = os.path.abspath(directory)
|
| 28 |
-
if not os.path.isdir(directory_path):
|
| 29 |
-
print(f"Directory not found: {directory_path}")
|
| 30 |
-
return final_items
|
| 31 |
-
|
| 32 |
-
files = os.listdir(directory_path)
|
| 33 |
-
for file in files:
|
| 34 |
-
file_path = os.path.join(directory_path, file)
|
| 35 |
-
try:
|
| 36 |
-
with open(file_path, 'r') as f:
|
| 37 |
-
last_part = os.path.basename(file_path)
|
| 38 |
-
file_content = f.read()
|
| 39 |
-
items = text_splitter2.create_documents([file_content])
|
| 40 |
-
for item in items:
|
| 41 |
-
item.metadata = last_part
|
| 42 |
-
final_items.extend(items)
|
| 43 |
-
except Exception as e:
|
| 44 |
-
print(f"Error reading file {file_path}: {e}")
|
| 45 |
-
|
| 46 |
-
return final_items
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|