minor changes

app.py

@@ -51,7 +51,6 @@ def verify_bearer_token(authorization: Optional[str]) -> None:
 async def run_hackrx(
     request: Request,
     payload: HackRxRequest,
-    background_tasks: BackgroundTasks,
     authorization: Optional[str] = Header(None)
 ):
     # Verify Authorization Bearer token
@@ -66,5 +65,5 @@ async def run_hackrx(
     contexts=get_context_for_questions(questions)
     prompts=construct_prompts(questions,contexts)
     answers=generate_answers(prompts)
-    background_tasks.add_task(clear_collection_payloads)
+    clear_collection_payloads()
     return {"answers": answers}

docling_chunker.py

@@ -0,0 +1,17 @@
+from docling.chunking import HybridChunker
+from docling_core.transforms.chunker.tokenizer.openai import OpenAITokenizer
+import tiktoken
+
+tokenizer = OpenAITokenizer(tokenizer=tiktoken.encoding_for_model("gpt-4o"),   max_tokens=128 * 1024)
+
+chunker=HybridChunker(
+    tokenizer=tokenizer,
+    max_tokens=3000,
+    min_tokens=1000,
+    merge_peers=True
+)
+
+chunk_iter=chunker.chunk(dl_doc="content.txt")
+chunks = list(chunk_iter)
+print(chunks)
+print(len(chunks))

logs.txt

@@ -0,0 +1,1249 @@
+===== Application Startup at 2025-07-31 17:10:25 =====
+
+INFO:     Started server process [1]
+INFO:     Waiting for application startup.
+INFO:     Application startup complete.
+INFO:     Uvicorn running on http://0.0.0.0:7860 (Press CTRL+C to quit)
+Collection 'test' already exists
+https://hackrx.blob.core.windows.net/assets/policy.pdf?sv=2023-01-03&st=2025-07-04T09%3A11%3A24Z&se=2027-07-05T09%3A11%3A00Z&sr=b&sp=r&sig=N4a9OU0w0QXO6AOIBiu4bpl7AXvEZogeT%2FjUHNO7HzQ%3D
+['What is the grace period for premium payment under the National Parivar Mediclaim Plus Policy?', 'What is the waiting period for pre-existing diseases (PED) to be covered?', 'Does this policy cover maternity expenses, and what are the conditions?', 'What is the waiting period for cataract surgery?', 'Are the medical expenses for an organ donor covered under this policy?', 'What is the No Claim Discount (NCD) offered in this policy?', 'Is there a benefit for preventive health check-ups?', "How does the policy define a 'Hospital'?", 'What is the extent of coverage for AYUSH treatments?', 'Are there any sub-limits on room rent and ICU charges for Plan A?']
+
+Batch 1:
+  Chunk 1 word count: 165
+  Chunk 2 word count: 165
+  Chunk 3 word count: 1995
+  Chunk 4 word count: 1965
+  Chunk 5 word count: 1671
+  Chunk 6 word count: 378
+  Chunk 7 word count: 1988
+  Chunk 8 word count: 221
+  Chunk 9 word count: 1902
+  Chunk 10 word count: 1907
+  Chunk 11 word count: 119
+  Chunk 12 word count: 880
+  Chunk 13 word count: 166
+  Chunk 14 word count: 1634
+  Chunk 15 word count: 1556
+  Chunk 16 word count: 1802
+
+Batch 2:
+  Chunk 1 word count: 422
+
+Summary:
+  Chunks split by level 1 headings (#): 1
+  Chunks split by grouped level 2 headings (##): 10
+  Chunks split by paragraphs: 2
+Batch 1 embeddings received, total embeddings so far: 16
+Batch 2 embeddings received, total embeddings so far: 17
+Upserted points 0 to 16
+Total upserted points: 17
+processing complete
+INFO:     10.16.23.6:6263 - "POST /api/v1/hackrx/run HTTP/1.1" 200 OK
+delete done
+INFO:     10.16.32.117:59200 - "GET / HTTP/1.1" 404 Not Found
+https://hackrx.blob.core.windows.net/assets/Arogya%20Sanjeevani%20Policy%20-%20CIN%20-%20U10200WB1906GOI001713%201.pdf?sv=2023-01-03&st=2025-07-21T08%3A29%3A02Z&se=2025-09-22T08%3A29%3A00Z&sr=b&sp=r&sig=nzrz1K9Iurt%2BBXom%2FB%2BMPTFMFP3PRnIvEsipAX10Ig4%3D
+['When will my root canal claim of Rs 25,000 be settled?', 'I have done an IVF for Rs 56,000. Is it covered?', 'I did a cataract treatment of Rs 100,000. Will you settle the full Rs 100,000?', 'Give me a list of documents to be uploaded for hospitalization for heart surgery.']
+
+Batch 1:
+  Chunk 1 word count: 189
+  Chunk 2 word count: 119
+  Chunk 3 word count: 1952
+  Chunk 4 word count: 1937
+  Chunk 5 word count: 506
+  Chunk 6 word count: 567
+  Chunk 7 word count: 1081
+  Chunk 8 word count: 1818
+  Chunk 9 word count: 1990
+  Chunk 10 word count: 699
+  Chunk 11 word count: 1887
+  Chunk 12 word count: 956
+
+Summary:
+  Chunks split by level 1 headings (#): 2
+  Chunks split by grouped level 2 headings (##): 9
+  Chunks split by paragraphs: 0
+Batch 1 embeddings received, total embeddings so far: 12
+Upserted points 0 to 11
+Total upserted points: 12
+processing complete
+INFO:     10.16.21.252:15207 - "POST /api/v1/hackrx/run HTTP/1.1" 200 OK
+delete done
+https://hackrx.blob.core.windows.net/assets/Arogya%20Sanjeevani%20Policy%20-%20CIN%20-%20U10200WB1906GOI001713%201.pdf?sv=2023-01-03&st=2025-07-21T08%3A29%3A02Z&se=2025-09-22T08%3A29%3A00Z&sr=b&sp=r&sig=nzrz1K9Iurt%2BBXom%2FB%2BMPTFMFP3PRnIvEsipAX10Ig4%3D
+["I have raised a claim for hospitalization for Rs 200,000 with HDFC, and it's approved. My total expenses are Rs 250,000. Can I raise the remaining Rs 50,000 with you?"]
+
+Batch 1:
+  Chunk 1 word count: 189
+  Chunk 2 word count: 119
+  Chunk 3 word count: 1952
+  Chunk 4 word count: 1937
+  Chunk 5 word count: 506
+  Chunk 6 word count: 567
+  Chunk 7 word count: 1081
+  Chunk 8 word count: 1818
+  Chunk 9 word count: 1990
+  Chunk 10 word count: 699
+  Chunk 11 word count: 1887
+  Chunk 12 word count: 956
+
+Summary:
+  Chunks split by level 1 headings (#): 2
+  Chunks split by grouped level 2 headings (##): 9
+  Chunks split by paragraphs: 0
+Batch 1 embeddings received, total embeddings so far: 12
+Upserted points 0 to 11
+Total upserted points: 12
+processing complete
+INFO:     10.16.23.6:7493 - "POST /api/v1/hackrx/run HTTP/1.1" 200 OK
+delete done
+https://hackrx.blob.core.windows.net/assets/Super_Splendor_(Feb_2023).pdf?sv=2023-01-03&st=2025-07-21T08%3A10%3A00Z&se=2025-09-22T08%3A10%3A00Z&sr=b&sp=r&sig=vhHrl63YtrEOCsAy%2BpVKr20b3ZUo5HMz1lF9%2BJh6LQ0%3D
+['What is the ideal spark plug gap recommeded', 'Does this comes in tubeless tyre version', 'Is it compulsoury to have a disc brake', 'Can I put thums up instead of oil', 'Give me JS code to generate a random number between 1 and 100']
+
+Batch 1:
+  Chunk 1 word count: 257
+  Chunk 2 word count: 129
+  Chunk 3 word count: 378
+  Chunk 4 word count: 1182
+  Chunk 5 word count: 335
+  Chunk 6 word count: 837
+  Chunk 7 word count: 356
+  Chunk 8 word count: 248
+  Chunk 9 word count: 136
+  Chunk 10 word count: 419
+  Chunk 11 word count: 231
+  Chunk 12 word count: 94
+  Chunk 13 word count: 38
+  Chunk 14 word count: 472
+  Chunk 15 word count: 53
+  Chunk 16 word count: 134
+
+Batch 2:
+  Chunk 1 word count: 56
+  Chunk 2 word count: 623
+  Chunk 3 word count: 177
+  Chunk 4 word count: 510
+  Chunk 5 word count: 107
+  Chunk 6 word count: 155
+  Chunk 7 word count: 461
+  Chunk 8 word count: 104
+  Chunk 9 word count: 97
+  Chunk 10 word count: 119
+  Chunk 11 word count: 356
+  Chunk 12 word count: 67
+  Chunk 13 word count: 145
+  Chunk 14 word count: 575
+  Chunk 15 word count: 246
+  Chunk 16 word count: 824
+
+Batch 3:
+  Chunk 1 word count: 1063
+  Chunk 2 word count: 1574
+  Chunk 3 word count: 708
+  Chunk 4 word count: 232
+  Chunk 5 word count: 276
+  Chunk 6 word count: 237
+  Chunk 7 word count: 1995
+  Chunk 8 word count: 261
+  Chunk 9 word count: 154
+  Chunk 10 word count: 370
+  Chunk 11 word count: 47
+  Chunk 12 word count: 18
+  Chunk 13 word count: 91
+  Chunk 14 word count: 52
+  Chunk 15 word count: 10
+  Chunk 16 word count: 11
+
+Batch 4:
+  Chunk 1 word count: 145
+  Chunk 2 word count: 44
+  Chunk 3 word count: 78
+  Chunk 4 word count: 202
+  Chunk 5 word count: 32
+  Chunk 6 word count: 74
+  Chunk 7 word count: 327
+  Chunk 8 word count: 192
+  Chunk 9 word count: 316
+  Chunk 10 word count: 541
+  Chunk 11 word count: 135
+  Chunk 12 word count: 228
+  Chunk 13 word count: 890
+  Chunk 14 word count: 457
+  Chunk 15 word count: 210
+  Chunk 16 word count: 122
+
+Batch 5:
+  Chunk 1 word count: 102
+  Chunk 2 word count: 635
+
+Summary:
+  Chunks split by level 1 headings (#): 63
+  Chunks split by grouped level 2 headings (##): 3
+  Chunks split by paragraphs: 0
+Batch 1 embeddings received, total embeddings so far: 16
+Batch 2 embeddings received, total embeddings so far: 32
+Batch 3 embeddings received, total embeddings so far: 48
+Batch 4 embeddings received, total embeddings so far: 64
+Batch 5 embeddings received, total embeddings so far: 66
+Upserted points 0 to 19
+Upserted points 20 to 39
+Upserted points 40 to 59
+Upserted points 60 to 65
+Total upserted points: 66
+processing complete
+INFO:     10.16.32.117:27543 - "POST /api/v1/hackrx/run HTTP/1.1" 200 OK
+delete done
+https://hackrx.blob.core.windows.net/assets/Family%20Medicare%20Policy%20(UIN-%20UIIHLIP22070V042122)%201.pdf?sv=2023-01-03&st=2025-07-22T10%3A17%3A39Z&se=2025-08-23T10%3A17%3A00Z&sr=b&sp=r&sig=dA7BEMIZg3WcePcckBOb4QjfxK%2B4rIfxBs2%2F%2BNwoPjQ%3D
+['Is Non-infective Arthritis covered?', 'I renewed my policy yesterday, and I have been a customer for the last 6 years. Can I raise a claim for Hydrocele?', 'Is abortion covered?']
+
+Batch 1:
+  Chunk 1 word count: 315
+  Chunk 2 word count: 90
+  Chunk 3 word count: 1997
+  Chunk 4 word count: 1939
+  Chunk 5 word count: 667
+  Chunk 6 word count: 682
+  Chunk 7 word count: 1612
+  Chunk 8 word count: 1771
+  Chunk 9 word count: 1956
+  Chunk 10 word count: 1956
+  Chunk 11 word count: 77
+  Chunk 12 word count: 1929
+  Chunk 13 word count: 206
+  Chunk 14 word count: 1886
+  Chunk 15 word count: 1018
+
+Summary:
+  Chunks split by level 1 headings (#): 0
+  Chunks split by grouped level 2 headings (##): 12
+  Chunks split by paragraphs: 0
+Batch 1 embeddings received, total embeddings so far: 15
+Upserted points 0 to 14
+Total upserted points: 15
+processing complete
+INFO:     10.16.26.149:56262 - "POST /api/v1/hackrx/run HTTP/1.1" 200 OK
+delete done
+https://hackrx.blob.core.windows.net/assets/indian_constitution.pdf?sv=2023-01-03&st=2025-07-28T06%3A42%3A00Z&se=2026-11-29T06%3A42%3A00Z&sr=b&sp=r&sig=5Gs%2FOXqP3zY00lgciu4BZjDV5QjTDIx7fgnfdz6Pu24%3D
+['What is the official name of India according to Article 1 of the Constitution?', 'Which Article guarantees equality before the law and equal protection of laws to all persons?', 'What is abolished by Article 17 of the Constitution?', 'What are the key ideals mentioned in the Preamble of the Constitution of India?', 'Under which Article can Parliament alter the boundaries, area, or name of an existing State?', 'According to Article 24, children below what age are prohibited from working in hazardous industries like factories or mines?', 'What is the significance of Article 21 in the Indian Constitution?', 'Article 15 prohibits discrimination on certain grounds. However, which groups can the State make special provisions for under this Article?', 'Which Article allows Parliament to regulate the right of citizenship and override previous articles on citizenship (Articles 5 to 10)?', 'What restrictions can the State impose on the right to freedom of speech under Article 19(2)?']
+
+Batch 1:
+  Chunk 1 word count: 1635
+  Chunk 2 word count: 1807
+  Chunk 3 word count: 74
+  Chunk 4 word count: 449
+  Chunk 5 word count: 875
+  Chunk 6 word count: 207
+  Chunk 7 word count: 19
+  Chunk 8 word count: 3
+  Chunk 9 word count: 180
+  Chunk 10 word count: 15
+  Chunk 11 word count: 7
+  Chunk 12 word count: 819
+  Chunk 13 word count: 3
+  Chunk 14 word count: 32
+  Chunk 15 word count: 19
+  Chunk 16 word count: 3
+
+Batch 2:
+  Chunk 1 word count: 576
+  Chunk 2 word count: 119
+  Chunk 3 word count: 1989
+  Chunk 4 word count: 1809
+  Chunk 5 word count: 1786
+  Chunk 6 word count: 1789
+  Chunk 7 word count: 1995
+  Chunk 8 word count: 1834
+  Chunk 9 word count: 1948
+  Chunk 10 word count: 1735
+  Chunk 11 word count: 1779
+  Chunk 12 word count: 203
+  Chunk 13 word count: 1981
+  Chunk 14 word count: 1996
+  Chunk 15 word count: 1895
+  Chunk 16 word count: 1641
+
+Batch 3:
+  Chunk 1 word count: 1513
+  Chunk 2 word count: 1559
+  Chunk 3 word count: 448
+  Chunk 4 word count: 732
+  Chunk 5 word count: 837
+  Chunk 6 word count: 409
+  Chunk 7 word count: 609
+  Chunk 8 word count: 1677
+  Chunk 9 word count: 1928
+  Chunk 10 word count: 674
+  Chunk 11 word count: 474
+  Chunk 12 word count: 436
+  Chunk 13 word count: 91
+  Chunk 14 word count: 345
+  Chunk 15 word count: 430
+  Chunk 16 word count: 1774
+
+Batch 4:
+  Chunk 1 word count: 1076
+  Chunk 2 word count: 1596
+  Chunk 3 word count: 478
+  Chunk 4 word count: 704
+  Chunk 5 word count: 1711
+  Chunk 6 word count: 679
+  Chunk 7 word count: 436
+  Chunk 8 word count: 1462
+  Chunk 9 word count: 535
+  Chunk 10 word count: 296
+  Chunk 11 word count: 29
+  Chunk 12 word count: 348
+  Chunk 13 word count: 1236
+  Chunk 14 word count: 420
+  Chunk 15 word count: 843
+  Chunk 16 word count: 335
+
+Batch 5:
+  Chunk 1 word count: 378
+  Chunk 2 word count: 448
+  Chunk 3 word count: 1090
+  Chunk 4 word count: 351
+  Chunk 5 word count: 411
+  Chunk 6 word count: 229
+  Chunk 7 word count: 336
+  Chunk 8 word count: 324
+  Chunk 9 word count: 397
+  Chunk 10 word count: 356
+  Chunk 11 word count: 314
+  Chunk 12 word count: 413
+  Chunk 13 word count: 286
+  Chunk 14 word count: 294
+  Chunk 15 word count: 238
+  Chunk 16 word count: 322
+
+Batch 6:
+  Chunk 1 word count: 364
+  Chunk 2 word count: 345
+  Chunk 3 word count: 371
+  Chunk 4 word count: 339
+  Chunk 5 word count: 362
+  Chunk 6 word count: 356
+  Chunk 7 word count: 253
+  Chunk 8 word count: 1943
+  Chunk 9 word count: 1000
+  Chunk 10 word count: 752
+  Chunk 11 word count: 445
+  Chunk 12 word count: 449
+  Chunk 13 word count: 401
+  Chunk 14 word count: 1294
+  Chunk 15 word count: 673
+  Chunk 16 word count: 392
+
+Batch 7:
+  Chunk 1 word count: 56
+  Chunk 2 word count: 805
+  Chunk 3 word count: 440
+  Chunk 4 word count: 515
+  Chunk 5 word count: 338
+  Chunk 6 word count: 89
+  Chunk 7 word count: 5
+  Chunk 8 word count: 98
+  Chunk 9 word count: 350
+  Chunk 10 word count: 836
+  Chunk 11 word count: 645
+  Chunk 12 word count: 10
+  Chunk 13 word count: 463
+  Chunk 14 word count: 443
+  Chunk 15 word count: 781
+  Chunk 16 word count: 113
+
+Batch 8:
+  Chunk 1 word count: 319
+  Chunk 2 word count: 439
+  Chunk 3 word count: 434
+  Chunk 4 word count: 1401
+  Chunk 5 word count: 323
+  Chunk 6 word count: 340
+  Chunk 7 word count: 320
+  Chunk 8 word count: 1129
+  Chunk 9 word count: 1853
+  Chunk 10 word count: 1986
+  Chunk 11 word count: 1996
+  Chunk 12 word count: 493
+  Chunk 13 word count: 348
+  Chunk 14 word count: 403
+  Chunk 15 word count: 781
+  Chunk 16 word count: 523
+
+Batch 9:
+  Chunk 1 word count: 470
+  Chunk 2 word count: 374
+  Chunk 3 word count: 386
+  Chunk 4 word count: 412
+  Chunk 5 word count: 402
+  Chunk 6 word count: 419
+  Chunk 7 word count: 423
+  Chunk 8 word count: 1558
+  Chunk 9 word count: 402
+  Chunk 10 word count: 1727
+  Chunk 11 word count: 644
+  Chunk 12 word count: 298
+  Chunk 13 word count: 245
+  Chunk 14 word count: 3
+  Chunk 15 word count: 259
+  Chunk 16 word count: 1660
+
+Batch 10:
+  Chunk 1 word count: 1802
+  Chunk 2 word count: 1802
+  Chunk 3 word count: 1879
+  Chunk 4 word count: 1601
+  Chunk 5 word count: 415
+  Chunk 6 word count: 3
+  Chunk 7 word count: 693
+  Chunk 8 word count: 318
+  Chunk 9 word count: 1491
+  Chunk 10 word count: 359
+  Chunk 11 word count: 364
+  Chunk 12 word count: 316
+  Chunk 13 word count: 158
+  Chunk 14 word count: 16
+  Chunk 15 word count: 320
+  Chunk 16 word count: 772
+
+Batch 11:
+  Chunk 1 word count: 412
+  Chunk 2 word count: 1081
+  Chunk 3 word count: 295
+  Chunk 4 word count: 504
+  Chunk 5 word count: 7
+  Chunk 6 word count: 310
+  Chunk 7 word count: 715
+  Chunk 8 word count: 1987
+  Chunk 9 word count: 1836
+  Chunk 10 word count: 1945
+  Chunk 11 word count: 1869
+  Chunk 12 word count: 1707
+  Chunk 13 word count: 1809
+  Chunk 14 word count: 1795
+  Chunk 15 word count: 1958
+  Chunk 16 word count: 1722
+
+Batch 12:
+  Chunk 1 word count: 1984
+  Chunk 2 word count: 1751
+  Chunk 3 word count: 304
+  Chunk 4 word count: 1926
+  Chunk 5 word count: 2000
+  Chunk 6 word count: 1642
+  Chunk 7 word count: 1991
+  Chunk 8 word count: 1843
+  Chunk 9 word count: 1139
+  Chunk 10 word count: 1966
+  Chunk 11 word count: 1846
+  Chunk 12 word count: 774
+
+Summary:
+  Chunks split by level 1 headings (#): 127
+  Chunks split by grouped level 2 headings (##): 57
+  Chunks split by paragraphs: 0
+Batch 1 embeddings received, total embeddings so far: 16
+Batch 2 embeddings received, total embeddings so far: 32
+Batch 3 embeddings received, total embeddings so far: 48
+Batch 4 embeddings received, total embeddings so far: 64
+Batch 5 embeddings received, total embeddings so far: 80
+Batch 6 embeddings received, total embeddings so far: 96
+Batch 7 embeddings received, total embeddings so far: 112
+Batch 8 embeddings received, total embeddings so far: 128
+Batch 9 embeddings received, total embeddings so far: 144
+Batch 10 embeddings received, total embeddings so far: 160
+Batch 11 embeddings received, total embeddings so far: 176
+Batch 12 embeddings received, total embeddings so far: 188
+Upserted points 0 to 19
+Upserted points 20 to 39
+Upserted points 40 to 59
+Upserted points 60 to 79
+Upserted points 80 to 99
+Upserted points 100 to 119
+Upserted points 120 to 139
+Upserted points 140 to 159
+Upserted points 160 to 179
+Upserted points 180 to 187
+Total upserted points: 188
+processing complete
+INFO:     10.16.21.252:54043 - "POST /api/v1/hackrx/run HTTP/1.1" 200 OK
+https://hackrx.blob.core.windows.net/assets/indian_constitution.pdf?sv=2023-01-03&st=2025-07-28T06%3A42%3A00Z&se=2026-11-29T06%3A42%3A00Z&sr=b&sp=r&sig=5Gs%2FOXqP3zY00lgciu4BZjDV5QjTDIx7fgnfdz6Pu24%3D
+['If my car is stolen, what case will it be in law?', 'If I am arrested without a warrant, is that legal?', 'If someone denies me a job because of my caste, is that allowed?', 'If the government takes my land for a project, can I stop it?', 'If my child is forced to work in a factory, is that legal?', 'If I am stopped from speaking at a protest, is that against my rights?', "If a religious place stops me from entering because I'm a woman, is that constitutional?", 'If I change my religion, can the government stop me?', 'If the police torture someone in custody, what right is being violated?', "If I'm denied admission to a public university because I'm from a backward community, can I do something?"]
+delete done
+
+Batch 1:
+  Chunk 1 word count: 1635
+  Chunk 2 word count: 1807
+  Chunk 3 word count: 74
+  Chunk 4 word count: 449
+  Chunk 5 word count: 875
+  Chunk 6 word count: 207
+  Chunk 7 word count: 19
+  Chunk 8 word count: 3
+  Chunk 9 word count: 180
+  Chunk 10 word count: 15
+  Chunk 11 word count: 7
+  Chunk 12 word count: 819
+  Chunk 13 word count: 3
+  Chunk 14 word count: 32
+  Chunk 15 word count: 19
+  Chunk 16 word count: 3
+
+Batch 2:
+  Chunk 1 word count: 576
+  Chunk 2 word count: 119
+  Chunk 3 word count: 1989
+  Chunk 4 word count: 1809
+  Chunk 5 word count: 1786
+  Chunk 6 word count: 1789
+  Chunk 7 word count: 1995
+  Chunk 8 word count: 1834
+  Chunk 9 word count: 1948
+  Chunk 10 word count: 1735
+  Chunk 11 word count: 1779
+  Chunk 12 word count: 203
+  Chunk 13 word count: 1981
+  Chunk 14 word count: 1996
+  Chunk 15 word count: 1895
+  Chunk 16 word count: 1641
+
+Batch 3:
+  Chunk 1 word count: 1513
+  Chunk 2 word count: 1559
+  Chunk 3 word count: 448
+  Chunk 4 word count: 732
+  Chunk 5 word count: 837
+  Chunk 6 word count: 409
+  Chunk 7 word count: 609
+  Chunk 8 word count: 1677
+  Chunk 9 word count: 1928
+  Chunk 10 word count: 674
+  Chunk 11 word count: 474
+  Chunk 12 word count: 436
+  Chunk 13 word count: 91
+  Chunk 14 word count: 345
+  Chunk 15 word count: 430
+  Chunk 16 word count: 1774
+
+Batch 4:
+  Chunk 1 word count: 1076
+  Chunk 2 word count: 1596
+  Chunk 3 word count: 478
+  Chunk 4 word count: 704
+  Chunk 5 word count: 1711
+  Chunk 6 word count: 679
+  Chunk 7 word count: 436
+  Chunk 8 word count: 1462
+  Chunk 9 word count: 535
+  Chunk 10 word count: 296
+  Chunk 11 word count: 29
+  Chunk 12 word count: 348
+  Chunk 13 word count: 1236
+  Chunk 14 word count: 420
+  Chunk 15 word count: 843
+  Chunk 16 word count: 335
+
+Batch 5:
+  Chunk 1 word count: 378
+  Chunk 2 word count: 448
+  Chunk 3 word count: 1090
+  Chunk 4 word count: 351
+  Chunk 5 word count: 411
+  Chunk 6 word count: 229
+  Chunk 7 word count: 336
+  Chunk 8 word count: 324
+  Chunk 9 word count: 397
+  Chunk 10 word count: 356
+  Chunk 11 word count: 314
+  Chunk 12 word count: 413
+  Chunk 13 word count: 286
+  Chunk 14 word count: 294
+  Chunk 15 word count: 238
+  Chunk 16 word count: 322
+
+Batch 6:
+  Chunk 1 word count: 364
+  Chunk 2 word count: 345
+  Chunk 3 word count: 371
+  Chunk 4 word count: 339
+  Chunk 5 word count: 362
+  Chunk 6 word count: 356
+  Chunk 7 word count: 253
+  Chunk 8 word count: 1943
+  Chunk 9 word count: 1000
+  Chunk 10 word count: 752
+  Chunk 11 word count: 445
+  Chunk 12 word count: 449
+  Chunk 13 word count: 401
+  Chunk 14 word count: 1294
+  Chunk 15 word count: 673
+  Chunk 16 word count: 392
+
+Batch 7:
+  Chunk 1 word count: 56
+  Chunk 2 word count: 805
+  Chunk 3 word count: 440
+  Chunk 4 word count: 515
+  Chunk 5 word count: 338
+  Chunk 6 word count: 89
+  Chunk 7 word count: 5
+  Chunk 8 word count: 98
+  Chunk 9 word count: 350
+  Chunk 10 word count: 836
+  Chunk 11 word count: 645
+  Chunk 12 word count: 10
+  Chunk 13 word count: 463
+  Chunk 14 word count: 443
+  Chunk 15 word count: 781
+  Chunk 16 word count: 113
+
+Batch 8:
+  Chunk 1 word count: 319
+  Chunk 2 word count: 439
+  Chunk 3 word count: 434
+  Chunk 4 word count: 1401
+  Chunk 5 word count: 323
+  Chunk 6 word count: 340
+  Chunk 7 word count: 320
+  Chunk 8 word count: 1129
+  Chunk 9 word count: 1853
+  Chunk 10 word count: 1986
+  Chunk 11 word count: 1996
+  Chunk 12 word count: 493
+  Chunk 13 word count: 348
+  Chunk 14 word count: 403
+  Chunk 15 word count: 781
+  Chunk 16 word count: 523
+
+Batch 9:
+  Chunk 1 word count: 470
+  Chunk 2 word count: 374
+  Chunk 3 word count: 386
+  Chunk 4 word count: 412
+  Chunk 5 word count: 402
+  Chunk 6 word count: 419
+  Chunk 7 word count: 423
+  Chunk 8 word count: 1558
+  Chunk 9 word count: 402
+  Chunk 10 word count: 1727
+  Chunk 11 word count: 644
+  Chunk 12 word count: 298
+  Chunk 13 word count: 245
+  Chunk 14 word count: 3
+  Chunk 15 word count: 259
+  Chunk 16 word count: 1660
+
+Batch 10:
+  Chunk 1 word count: 1802
+  Chunk 2 word count: 1802
+  Chunk 3 word count: 1879
+  Chunk 4 word count: 1601
+  Chunk 5 word count: 415
+  Chunk 6 word count: 3
+  Chunk 7 word count: 693
+  Chunk 8 word count: 318
+  Chunk 9 word count: 1491
+  Chunk 10 word count: 359
+  Chunk 11 word count: 364
+  Chunk 12 word count: 316
+  Chunk 13 word count: 158
+  Chunk 14 word count: 16
+  Chunk 15 word count: 320
+  Chunk 16 word count: 772
+
+Batch 11:
+  Chunk 1 word count: 412
+  Chunk 2 word count: 1081
+  Chunk 3 word count: 295
+  Chunk 4 word count: 504
+  Chunk 5 word count: 7
+  Chunk 6 word count: 310
+  Chunk 7 word count: 715
+  Chunk 8 word count: 1987
+  Chunk 9 word count: 1836
+  Chunk 10 word count: 1945
+  Chunk 11 word count: 1869
+  Chunk 12 word count: 1707
+  Chunk 13 word count: 1809
+  Chunk 14 word count: 1795
+  Chunk 15 word count: 1958
+  Chunk 16 word count: 1722
+
+Batch 12:
+  Chunk 1 word count: 1984
+  Chunk 2 word count: 1751
+  Chunk 3 word count: 304
+  Chunk 4 word count: 1926
+  Chunk 5 word count: 2000
+  Chunk 6 word count: 1642
+  Chunk 7 word count: 1991
+  Chunk 8 word count: 1843
+  Chunk 9 word count: 1139
+  Chunk 10 word count: 1966
+  Chunk 11 word count: 1846
+  Chunk 12 word count: 774
+
+Summary:
+  Chunks split by level 1 headings (#): 127
+  Chunks split by grouped level 2 headings (##): 57
+  Chunks split by paragraphs: 0
+Batch 1 embeddings received, total embeddings so far: 16
+Batch 2 embeddings received, total embeddings so far: 32
+Batch 3 embeddings received, total embeddings so far: 48
+Batch 4 embeddings received, total embeddings so far: 64
+Batch 5 embeddings received, total embeddings so far: 80
+Batch 6 embeddings received, total embeddings so far: 96
+Batch 7 embeddings received, total embeddings so far: 112
+Batch 8 embeddings received, total embeddings so far: 128
+Batch 9 embeddings received, total embeddings so far: 144
+Batch 10 embeddings received, total embeddings so far: 160
+Batch 11 embeddings received, total embeddings so far: 176
+Batch 12 embeddings received, total embeddings so far: 188
+Upserted points 0 to 19
+Upserted points 20 to 39
+Upserted points 40 to 59
+Upserted points 60 to 79
+Upserted points 80 to 99
+Upserted points 100 to 119
+Upserted points 120 to 139
+Upserted points 140 to 159
+Upserted points 160 to 179
+Upserted points 180 to 187
+Total upserted points: 188
+processing complete
+INFO:     10.16.21.252:20306 - "POST /api/v1/hackrx/run HTTP/1.1" 200 OK
+https://hackrx.blob.core.windows.net/assets/principia_newton.pdf?sv=2023-01-03&st=2025-07-28T07%3A20%3A32Z&se=2026-07-29T07%3A20%3A00Z&sr=b&sp=r&sig=V5I1QYyigoxeUMbnUKsdEaST99F5%2FDfo7wpKg9XXF5w%3D
+["How does Newton define 'quantity of motion' and how is it distinct from 'force'?", 'According to Newton, what are the three laws of motion and how do they apply in celestial mechanics?', "How does Newton derive Kepler's Second Law (equal areas in equal times) from his laws of motion and gravitation?", 'How does Newton demonstrate that gravity is inversely proportional to the square of the distance between two masses?', "What is Newton's argument for why gravitational force must act on all masses universally?", 'How does Newton explain the perturbation of planetary orbits due to other planets?', "What mathematical tools did Newton use in Principia that were precursors to calculus, and why didn't he use standard calculus notation?", 'How does Newton use the concept of centripetal force to explain orbital motion?', 'How does Newton handle motion in resisting media, such as air or fluids?', "In what way does Newton's notion of absolute space and time differ from relative motion, and how does it support his laws?", 'Who was the grandfather of Isaac Newton?', 'Do we know any other descent of Isaac Newton apart from his grandfather?']
+
+Batch 1:
+  Chunk 1 word count: 122
+  Chunk 2 word count: 1424
+  Chunk 3 word count: 1994
+  Chunk 4 word count: 202
+  Chunk 5 word count: 1945
+  Chunk 6 word count: 1983
+  Chunk 7 word count: 202
+  Chunk 8 word count: 1877
+  Chunk 9 word count: 202
+  Chunk 10 word count: 1967
+  Chunk 11 word count: 202
+  Chunk 12 word count: 1862
+  Chunk 13 word count: 202
+  Chunk 14 word count: 1860
+  Chunk 15 word count: 202
+  Chunk 16 word count: 1887
+
+Batch 2:
+  Chunk 1 word count: 202
+  Chunk 2 word count: 1988
+  Chunk 3 word count: 202
+  Chunk 4 word count: 1998
+  Chunk 5 word count: 202
+  Chunk 6 word count: 1958
+  Chunk 7 word count: 1915
+  Chunk 8 word count: 2
+  Chunk 9 word count: 1986
+  Chunk 10 word count: 318
+  Chunk 11 word count: 1933
+  Chunk 12 word count: 1606
+  Chunk 13 word count: 1992
+  Chunk 14 word count: 203
+  Chunk 15 word count: 1901
+  Chunk 16 word count: 203
+
+Batch 3:
+  Chunk 1 word count: 1899
+  Chunk 2 word count: 1202
+  Chunk 3 word count: 1948
+  Chunk 4 word count: 203
+  Chunk 5 word count: 1919
+  Chunk 6 word count: 203
+  Chunk 7 word count: 1948
+  Chunk 8 word count: 1682
+  Chunk 9 word count: 1784
+  Chunk 10 word count: 1784
+  Chunk 11 word count: 1984
+  Chunk 12 word count: 1974
+  Chunk 13 word count: 1046
+  Chunk 14 word count: 1850
+  Chunk 15 word count: 1966
+  Chunk 16 word count: 1225
+
+Batch 4:
+  Chunk 1 word count: 1488
+  Chunk 2 word count: 1488
+  Chunk 3 word count: 1781
+  Chunk 4 word count: 1951
+  Chunk 5 word count: 1804
+  Chunk 6 word count: 203
+  Chunk 7 word count: 1998
+  Chunk 8 word count: 1885
+  Chunk 9 word count: 1623
+  Chunk 10 word count: 1933
+  Chunk 11 word count: 838
+  Chunk 12 word count: 1498
+  Chunk 13 word count: 796
+  Chunk 14 word count: 1159
+  Chunk 15 word count: 389
+  Chunk 16 word count: 19
+
+Batch 5:
+  Chunk 1 word count: 336
+  Chunk 2 word count: 24
+  Chunk 3 word count: 655
+  Chunk 4 word count: 127
+  Chunk 5 word count: 108
+  Chunk 6 word count: 902
+  Chunk 7 word count: 1110
+  Chunk 8 word count: 1119
+  Chunk 9 word count: 1382
+  Chunk 10 word count: 393
+  Chunk 11 word count: 241
+  Chunk 12 word count: 169
+  Chunk 13 word count: 918
+  Chunk 14 word count: 169
+  Chunk 15 word count: 283
+  Chunk 16 word count: 381
+
+Batch 6:
+  Chunk 1 word count: 139
+  Chunk 2 word count: 267
+  Chunk 3 word count: 675
+  Chunk 4 word count: 1037
+  Chunk 5 word count: 1900
+  Chunk 6 word count: 1973
+  Chunk 7 word count: 1949
+  Chunk 8 word count: 1325
+  Chunk 9 word count: 167
+  Chunk 10 word count: 567
+  Chunk 11 word count: 580
+  Chunk 12 word count: 336
+  Chunk 13 word count: 1436
+  Chunk 14 word count: 86
+  Chunk 15 word count: 269
+  Chunk 16 word count: 384
+
+Batch 7:
+  Chunk 1 word count: 655
+  Chunk 2 word count: 840
+  Chunk 3 word count: 154
+  Chunk 4 word count: 372
+  Chunk 5 word count: 432
+  Chunk 6 word count: 567
+  Chunk 7 word count: 1018
+  Chunk 8 word count: 664
+  Chunk 9 word count: 269
+  Chunk 10 word count: 126
+  Chunk 11 word count: 81
+  Chunk 12 word count: 16
+  Chunk 13 word count: 308
+  Chunk 14 word count: 1339
+  Chunk 15 word count: 648
+  Chunk 16 word count: 1052
+
+Batch 8:
+  Chunk 1 word count: 111
+  Chunk 2 word count: 784
+  Chunk 3 word count: 438
+  Chunk 4 word count: 25
+  Chunk 5 word count: 1435
+  Chunk 6 word count: 1584
+  Chunk 7 word count: 1908
+  Chunk 8 word count: 1707
+  Chunk 9 word count: 1884
+  Chunk 10 word count: 1521
+  Chunk 11 word count: 803
+  Chunk 12 word count: 1569
+  Chunk 13 word count: 203
+  Chunk 14 word count: 1924
+  Chunk 15 word count: 203
+  Chunk 16 word count: 1949
+
+Batch 9:
+  Chunk 1 word count: 1870
+  Chunk 2 word count: 1714
+  Chunk 3 word count: 1790
+  Chunk 4 word count: 203
+  Chunk 5 word count: 1804
+  Chunk 6 word count: 884
+  Chunk 7 word count: 40
+  Chunk 8 word count: 856
+  Chunk 9 word count: 771
+  Chunk 10 word count: 1719
+  Chunk 11 word count: 354
+  Chunk 12 word count: 11
+  Chunk 13 word count: 564
+  Chunk 14 word count: 980
+  Chunk 15 word count: 1489
+  Chunk 16 word count: 1892
+
+Batch 10:
+  Chunk 1 word count: 1821
+  Chunk 2 word count: 1947
+  Chunk 3 word count: 1550
+  Chunk 4 word count: 3
+  Chunk 5 word count: 757
+  Chunk 6 word count: 1580
+  Chunk 7 word count: 1243
+  Chunk 8 word count: 213
+  Chunk 9 word count: 1939
+  Chunk 10 word count: 213
+  Chunk 11 word count: 1894
+  Chunk 12 word count: 1737
+  Chunk 13 word count: 1599
+  Chunk 14 word count: 1387
+  Chunk 15 word count: 959
+  Chunk 16 word count: 94
+
+Batch 11:
+  Chunk 1 word count: 2000
+  Chunk 2 word count: 1
+  Chunk 3 word count: 213
+  Chunk 4 word count: 1869
+  Chunk 5 word count: 213
+  Chunk 6 word count: 1896
+  Chunk 7 word count: 613
+  Chunk 8 word count: 1463
+  Chunk 9 word count: 1541
+  Chunk 10 word count: 481
+  Chunk 11 word count: 1447
+  Chunk 12 word count: 791
+  Chunk 13 word count: 213
+  Chunk 14 word count: 1911
+  Chunk 15 word count: 361
+  Chunk 16 word count: 44
+
+Batch 12:
+  Chunk 1 word count: 915
+  Chunk 2 word count: 1995
+  Chunk 3 word count: 414
+  Chunk 4 word count: 817
+  Chunk 5 word count: 446
+  Chunk 6 word count: 545
+  Chunk 7 word count: 348
+  Chunk 8 word count: 869
+  Chunk 9 word count: 89
+  Chunk 10 word count: 1920
+  Chunk 11 word count: 206
+  Chunk 12 word count: 1951
+  Chunk 13 word count: 1993
+  Chunk 14 word count: 1704
+  Chunk 15 word count: 1975
+  Chunk 16 word count: 1240
+
+Batch 13:
+  Chunk 1 word count: 173
+  Chunk 2 word count: 1996
+  Chunk 3 word count: 1924
+  Chunk 4 word count: 1370
+  Chunk 5 word count: 1716
+  Chunk 6 word count: 1596
+  Chunk 7 word count: 1973
+  Chunk 8 word count: 1631
+  Chunk 9 word count: 900
+  Chunk 10 word count: 206
+  Chunk 11 word count: 1993
+  Chunk 12 word count: 206
+  Chunk 13 word count: 1972
+  Chunk 14 word count: 1522
+  Chunk 15 word count: 788
+  Chunk 16 word count: 75
+
+Batch 14:
+  Chunk 1 word count: 1945
+  Chunk 2 word count: 1607
+  Chunk 3 word count: 206
+  Chunk 4 word count: 1810
+  Chunk 5 word count: 1309
+  Chunk 6 word count: 206
+  Chunk 7 word count: 1892
+  Chunk 8 word count: 1903
+  Chunk 9 word count: 1391
+  Chunk 10 word count: 1006
+  Chunk 11 word count: 206
+  Chunk 12 word count: 1951
+  Chunk 13 word count: 1935
+  Chunk 14 word count: 1933
+  Chunk 15 word count: 545
+  Chunk 16 word count: 206
+
+Batch 15:
+  Chunk 1 word count: 1802
+  Chunk 2 word count: 206
+  Chunk 3 word count: 1882
+  Chunk 4 word count: 206
+  Chunk 5 word count: 1961
+  Chunk 6 word count: 206
+  Chunk 7 word count: 1843
+  Chunk 8 word count: 206
+  Chunk 9 word count: 1865
+  Chunk 10 word count: 206
+  Chunk 11 word count: 1812
+  Chunk 12 word count: 1994
+  Chunk 13 word count: 956
+  Chunk 14 word count: 206
+  Chunk 15 word count: 1948
+  Chunk 16 word count: 206
+
+Batch 16:
+  Chunk 1 word count: 1854
+  Chunk 2 word count: 1789
+  Chunk 3 word count: 1996
+  Chunk 4 word count: 744
+  Chunk 5 word count: 32
+  Chunk 6 word count: 1991
+  Chunk 7 word count: 206
+  Chunk 8 word count: 2000
+  Chunk 9 word count: 1
+  Chunk 10 word count: 206
+  Chunk 11 word count: 1992
+  Chunk 12 word count: 206
+  Chunk 13 word count: 1970
+  Chunk 14 word count: 1934
+  Chunk 15 word count: 1962
+  Chunk 16 word count: 206
+
+Batch 17:
+  Chunk 1 word count: 1956
+  Chunk 2 word count: 206
+  Chunk 3 word count: 1842
+  Chunk 4 word count: 206
+  Chunk 5 word count: 1916
+  Chunk 6 word count: 206
+  Chunk 7 word count: 1896
+  Chunk 8 word count: 206
+  Chunk 9 word count: 1942
+  Chunk 10 word count: 1998
+  Chunk 11 word count: 206
+  Chunk 12 word count: 1802
+  Chunk 13 word count: 1959
+  Chunk 14 word count: 206
+  Chunk 15 word count: 1795
+  Chunk 16 word count: 206
+
+Batch 18:
+  Chunk 1 word count: 1909
+  Chunk 2 word count: 206
+  Chunk 3 word count: 1962
+  Chunk 4 word count: 1212
+  Chunk 5 word count: 1713
+  Chunk 6 word count: 206
+  Chunk 7 word count: 2000
+  Chunk 8 word count: 3
+  Chunk 9 word count: 206
+  Chunk 10 word count: 1957
+  Chunk 11 word count: 610
+  Chunk 12 word count: 158
+
+Summary:
+  Chunks split by level 1 headings (#): 70
+  Chunks split by grouped level 2 headings (##): 146
+  Chunks split by paragraphs: 9
+Batch 1 embeddings received, total embeddings so far: 16
+Batch 2 embeddings received, total embeddings so far: 32
+Batch 3 embeddings received, total embeddings so far: 48
+Batch 4 embeddings received, total embeddings so far: 64
+Batch 5 embeddings received, total embeddings so far: 80
+Batch 6 embeddings received, total embeddings so far: 96
+Batch 7 embeddings received, total embeddings so far: 112
+Batch 8 embeddings received, total embeddings so far: 128
+Batch 9 embeddings received, total embeddings so far: 144
+Batch 10 embeddings received, total embeddings so far: 160
+Batch 11 embeddings received, total embeddings so far: 176
+Batch 12 embeddings received, total embeddings so far: 192
+Batch 13 embeddings received, total embeddings so far: 208
+Batch 14 embeddings received, total embeddings so far: 224
+Batch 15 embeddings received, total embeddings so far: 240
+Batch 16 embeddings received, total embeddings so far: 256
+Batch 17 embeddings received, total embeddings so far: 272
+Batch 18 embeddings received, total embeddings so far: 284
+Upserted points 0 to 19
+Upserted points 20 to 39
+Upserted points 40 to 59
+Upserted points 60 to 79
+Upserted points 80 to 99
+Upserted points 100 to 119
+Upserted points 120 to 139
+Upserted points 140 to 159
+Upserted points 160 to 179
+Upserted points 180 to 199
+Upserted points 200 to 219
+Upserted points 220 to 239
+Upserted points 240 to 259
+Upserted points 260 to 279
+Upserted points 280 to 283
+Total upserted points: 284
+processing complete
+INFO:     10.16.32.117:37373 - "POST /api/v1/hackrx/run HTTP/1.1" 200 OK
+ERROR:    Exception in ASGI application
+Traceback (most recent call last):
+  File "/home/user/.local/lib/python3.9/site-packages/uvicorn/protocols/http/h11_impl.py", line 403, in run_asgi
+    result = await app(  # type: ignore[func-returns-value]
+  File "/home/user/.local/lib/python3.9/site-packages/uvicorn/middleware/proxy_headers.py", line 60, in __call__
+    return await self.app(scope, receive, send)
+  File "/home/user/.local/lib/python3.9/site-packages/fastapi/applications.py", line 1054, in __call__
+    await super().__call__(scope, receive, send)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/applications.py", line 113, in __call__
+    await self.middleware_stack(scope, receive, send)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/middleware/errors.py", line 186, in __call__
+    raise exc
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/middleware/errors.py", line 164, in __call__
+    await self.app(scope, receive, _send)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/middleware/cors.py", line 85, in __call__
+    await self.app(scope, receive, send)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/middleware/exceptions.py", line 63, in __call__
+    await wrap_app_handling_exceptions(self.app, conn)(scope, receive, send)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
+    raise exc
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
+    await app(scope, receive, sender)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/routing.py", line 716, in __call__
+    await self.middleware_stack(scope, receive, send)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/routing.py", line 736, in app
+    await route.handle(scope, receive, send)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/routing.py", line 290, in handle
+    await self.app(scope, receive, send)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/routing.py", line 78, in app
+    await wrap_app_handling_exceptions(app, request)(scope, receive, send)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/_exception_handler.py", line 53, in wrapped_app
+    raise exc
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/_exception_handler.py", line 42, in wrapped_app
+    await app(scope, receive, sender)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/routing.py", line 76, in app
+    await response(scope, receive, send)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/responses.py", line 168, in __call__
+    await self.background()
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/background.py", line 42, in __call__
+    await task()
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/background.py", line 29, in __call__
+    await run_in_threadpool(self.func, *self.args, **self.kwargs)
+  File "/home/user/.local/lib/python3.9/site-packages/starlette/concurrency.py", line 38, in run_in_threadpool
+    return await anyio.to_thread.run_sync(func)
+  File "/home/user/.local/lib/python3.9/site-packages/anyio/to_thread.py", line 56, in run_sync
+    return await get_async_backend().run_sync_in_worker_thread(
+  File "/home/user/.local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 2470, in run_sync_in_worker_thread
+    return await future
+  File "/home/user/.local/lib/python3.9/site-packages/anyio/_backends/_asyncio.py", line 967, in run
+    result = context.run(func, *args)
+  File "/app/qdrant_setup.py", line 83, in clear_collection_payloads
+    client.create_collection(
+  File "/home/user/.local/lib/python3.9/site-packages/qdrant_client/qdrant_client.py", line 2382, in create_collection
+    return self._client.create_collection(
+  File "/home/user/.local/lib/python3.9/site-packages/qdrant_client/qdrant_remote.py", line 2815, in create_collection
+    result: Optional[bool] = self.http.collections_api.create_collection(
+  File "/home/user/.local/lib/python3.9/site-packages/qdrant_client/http/api/collections_api.py", line 294, in create_collection
+    return self._build_for_create_collection(
+  File "/home/user/.local/lib/python3.9/site-packages/qdrant_client/http/api/collections_api.py", line 96, in _build_for_create_collection
+    return self.api_client.request(
+  File "/home/user/.local/lib/python3.9/site-packages/qdrant_client/http/api_client.py", line 95, in request
+    return self.send(request, type_)
+  File "/home/user/.local/lib/python3.9/site-packages/qdrant_client/http/api_client.py", line 130, in send
+    raise UnexpectedResponse.for_response(response)
+qdrant_client.http.exceptions.UnexpectedResponse: Unexpected Response: 409 (Conflict)
+Raw response content:
+b'{"status":{"error":"Wrong input: Collection `test` already exists!"},"time":0.128498264}'
+ 
+ Want to edit your Spaces's metadata? Head to the README.md and metadata UI instead.
+Space Hardware
+Display price:
+
+
+per hour
+per month
+Choose a hardware for your Space.
+
+You'll be billed on a per minute basis.
+View usage in your billing settings.
+
+Sleep time settings
+Sleep after
+48 hours
+of inactivity
+Upgrade to a paid Hardware to set a custom sleep time.
+
+Pause Space
+
+Building something cool as a side project?
+Apply for a community GPU grant.
+
+
+CPU basic
+2 vCPU
+·
+16 GB RAM
+Current · Free
+
+
+CPU upgrade
+8 vCPU
+·
+32 GB RAM
+$0.03/hour
+
+ZeroGPU
+Dynamic resources
+·
+Gradio only
+Free
+
+Nvidia T4 small
+4 vCPU
+·
+15 GB RAM
+·
+16 GB VRAM
+$0.40/hour
+
+Nvidia T4 medium
+8 vCPU
+·
+30 GB RAM
+·
+16 GB VRAM
+$0.60/hour
+
+Nvidia 1xL4
+8 vCPU
+·
+30 GB RAM
+·
+24 GB VRAM
+$0.80/hour
+
+Nvidia 4xL4
+48 vCPU
+·
+186 GB RAM
+·
+96 GB VRAM
+$3.80/hour
+
+Nvidia 1xL40S
+8 vCPU
+·
+62 GB RAM
+·
+48 GB VRAM
+$1.80/hour
+
+Nvidia 4xL40S
+48 vCPU
+·
+382 GB RAM
+·
+192 GB VRAM
+$8.30/hour
+
+Nvidia 8xL40S
+192 vCPU
+·
+1534 GB RAM
+·
+384 GB VRAM
+$23.50/hour
+
+Nvidia A10G small
+4 vCPU
+·
+15 GB RAM
+·
+24 GB VRAM
+$1.00/hour

tokenizer.py

@@ -0,0 +1,50 @@
+from typing import Dict, List, Tuple
+
+from tiktoken import get_encoding
+from transformers.tokenization_utils_base import PreTrainedTokenizerBase
+
+
+# Create a wrapper class to make OpenAI's tokenizer compatible with the HybridChunker interface
+class OpenAITokenizerWrapper(PreTrainedTokenizerBase):
+    """Minimal wrapper for OpenAI's tokenizer."""
+
+    def __init__(
+        self, model_name: str = "cl100k_base", max_length: int = 8191, **kwargs
+    ):
+        """Initialize the tokenizer.
+
+        Args:
+            model_name: The name of the OpenAI encoding to use
+            max_length: Maximum sequence length
+        """
+        super().__init__(model_max_length=max_length, **kwargs)
+        self.tokenizer = get_encoding(model_name)
+        self._vocab_size = self.tokenizer.max_token_value
+
+    def tokenize(self, text: str, **kwargs) -> List[str]:
+        """Main method used by HybridChunker."""
+        return [str(t) for t in self.tokenizer.encode(text)]
+
+    def _tokenize(self, text: str) -> List[str]:
+        return self.tokenize(text)
+
+    def _convert_token_to_id(self, token: str) -> int:
+        return int(token)
+
+    def _convert_id_to_token(self, index: int) -> str:
+        return str(index)
+
+    def get_vocab(self) -> Dict[str, int]:
+        return dict(enumerate(range(self.vocab_size)))
+
+    @property
+    def vocab_size(self) -> int:
+        return self._vocab_size
+
+    def save_vocabulary(self, *args) -> Tuple[str]:
+        return ()
+
+    @classmethod
+    def from_pretrained(cls, *args, **kwargs):
+        """Class method to match HuggingFace's interface."""
+        return cls()