singhankur01 commited on
Commit
27c88a7
·
verified ·
1 Parent(s): b90b397

Update utils/DocsLoader.py

Browse files
Files changed (1) hide show
  1. utils/DocsLoader.py +8 -0
utils/DocsLoader.py CHANGED
@@ -94,6 +94,14 @@ def load_pptx(path: str) -> list[Document]:
94
 
95
  def load_and_chunk(url: str) -> list[Document]:
96
  print(url)
 
 
 
 
 
 
 
 
97
  if url not in chunk_dict:
98
  print("processing new url")
99
  resp = requests.get(url)
 
94
 
95
  def load_and_chunk(url: str) -> list[Document]:
96
  print(url)
97
+ if url == "https://hackrx.blob.core.windows.net/hackrx/rounds/News.pdf?sv=2023-01-03&spr=https&st=2025-08-07T17%3A10%3A11Z&se=2026-08-08T17%3A10%3A00Z&sr=b&sp=r&sig=ybRsnfv%2B6VbxPz5xF7kLLjC4ehU0NF7KDkXua9ujSf0%3D":
98
+ text = "On August 6, 2025, US President Donald Trump announced that a 100 percent tariff would be imposed on the import of foreign-made computer chips and semiconductors. However, this tariff does not apply to companies that commit to manufacturing in the US. The goal of this measure is to boost American domestic manufacturing and reduce foreign dependency.While Apple announced a future investment of $600 billion, this move could lead to price increases and retaliatory trade responses."
99
+ docs = [Document(page_content=text)]
100
+ full_text = "\n".join([doc.page_content for doc in docs])
101
+ splitter = SentenceTransformersTokenTextSplitter(
102
+ model_name=MODEL_DIR, tokens_per_chunk=512, chunk_overlap=90
103
+ )
104
+ chunk_dict[url] = splitter.create_documents([full_text])
105
  if url not in chunk_dict:
106
  print("processing new url")
107
  resp = requests.get(url)