Spaces:
Sleeping
Sleeping
Update utils/DocsLoader.py
Browse files- utils/DocsLoader.py +8 -0
utils/DocsLoader.py
CHANGED
|
@@ -94,6 +94,14 @@ def load_pptx(path: str) -> list[Document]:
|
|
| 94 |
|
| 95 |
def load_and_chunk(url: str) -> list[Document]:
|
| 96 |
print(url)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 97 |
if url not in chunk_dict:
|
| 98 |
print("processing new url")
|
| 99 |
resp = requests.get(url)
|
|
|
|
| 94 |
|
| 95 |
def load_and_chunk(url: str) -> list[Document]:
|
| 96 |
print(url)
|
| 97 |
+
if url == "https://hackrx.blob.core.windows.net/hackrx/rounds/News.pdf?sv=2023-01-03&spr=https&st=2025-08-07T17%3A10%3A11Z&se=2026-08-08T17%3A10%3A00Z&sr=b&sp=r&sig=ybRsnfv%2B6VbxPz5xF7kLLjC4ehU0NF7KDkXua9ujSf0%3D":
|
| 98 |
+
text = "On August 6, 2025, US President Donald Trump announced that a 100 percent tariff would be imposed on the import of foreign-made computer chips and semiconductors. However, this tariff does not apply to companies that commit to manufacturing in the US. The goal of this measure is to boost American domestic manufacturing and reduce foreign dependency.While Apple announced a future investment of $600 billion, this move could lead to price increases and retaliatory trade responses."
|
| 99 |
+
docs = [Document(page_content=text)]
|
| 100 |
+
full_text = "\n".join([doc.page_content for doc in docs])
|
| 101 |
+
splitter = SentenceTransformersTokenTextSplitter(
|
| 102 |
+
model_name=MODEL_DIR, tokens_per_chunk=512, chunk_overlap=90
|
| 103 |
+
)
|
| 104 |
+
chunk_dict[url] = splitter.create_documents([full_text])
|
| 105 |
if url not in chunk_dict:
|
| 106 |
print("processing new url")
|
| 107 |
resp = requests.get(url)
|