Merge dev_new to main (improve retriever and add querying)

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+software/** filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -26,6 +26,10 @@ test/users/user_accounts.csv
 .env.prd
 .env.example
 
+CLAUDE.md
+
+/experiments
+src/rag/experiments/
 erd/
 playground/
 playground_retriever.py
@@ -33,4 +37,6 @@ playground_chat.py
 playground_flush_cache.py
 playground_create_user.py
 API_CONTRACT.md
-context_engineering/
+context_engineering/
+sample_file/
+test_tesseract.py

.vscode/launch.json

@@ -0,0 +1,25 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "DataEyond: FastAPI (debug)",
+            "type": "debugpy",
+            "request": "launch",
+            "module": "uvicorn",
+            "args": [
+                "main:app",
+                "--host", "0.0.0.0",
+                "--port", "7860",
+                "--reload"
+            ],
+            "jinja": true,
+            "justMyCode": true,
+            "envFile": "${workspaceFolder}/.env",
+            "console": "integratedTerminal",
+            "cwd": "${workspaceFolder}"
+        }
+    ]
+}

Dockerfile

@@ -12,6 +12,8 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
     libpq-dev \
     gcc \
     libgomp1 \
+    tesseract-ocr \
+    poppler-utils \
     && rm -rf /var/lib/apt/lists/*
 
 RUN addgroup --system app && \

README.md

@@ -11,6 +11,8 @@ short_description: AI Agent core service
 Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
 
 
+# Agentic Service Data Eyond
+
 How to run:
 `uv run --no-sync uvicorn main:app --host 0.0.0.0 --port 7860`
 

main.py

@@ -10,6 +10,7 @@ from src.api.v1.chat import router as chat_router
 from src.api.v1.room import router as room_router
 from src.api.v1.users import router as users_router
 from src.api.v1.knowledge import router as knowledge_router
+from src.api.v1.db_client import router as db_client_router
 from src.db.postgres.init_db import init_db
 import uvicorn
 
@@ -35,6 +36,7 @@ app.include_router(document_router)
 app.include_router(knowledge_router)
 app.include_router(room_router)
 app.include_router(chat_router)
+app.include_router(db_client_router)
 
 
 @app.on_event("startup")

pyproject.toml

@@ -79,6 +79,17 @@ dependencies = [
     "jsonpatch>=1.33",
     "pymongo>=4.14.0",
     "psycopg2>=2.9.11",
+    # --- SQL parsing / guardrails ---
+    "sqlglot>=25.0.0",
+    # --- User-DB connectors (db_pipeline) ---
+    "pymysql>=1.1.1",
+    "pymssql>=2.3.0",
+    "sqlalchemy-bigquery>=1.11.0",
+    "snowflake-sqlalchemy>=1.7.0",
+    # --- OCR (pdf processing) ---
+    "pdf2image>=1.17.0",
+    "pytesseract>=0.3.13",
+    "pypdf2>=3.0.1",
 ]
 
 [project.optional-dependencies]
@@ -92,16 +103,6 @@ dev = [
     "pre-commit==4.0.1",
 ]
 
-[tool.uv]
-dev-dependencies = [
-    "pytest==8.3.4",
-    "pytest-asyncio==0.24.0",
-    "pytest-cov==6.0.0",
-    "ruff==0.8.4",
-    "mypy==1.13.0",
-    "pre-commit==4.0.1",
-]
-
 [tool.hatch.build.targets.wheel]
 packages = ["src/agent_service"]
 

software/Tesseract-OCR/ambiguous_words.1.html

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b9e122cc6b60a9515b253a99321354ae2de2d5886f72ee6e88841c38f111ff86
+size 17823

software/Tesseract-OCR/ambiguous_words.exe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:85c23ac33fa8235cf5f3adaf87b97c506a56e71a96dd0f7bb83b373c029c2ae5
+size 1066496

software/Tesseract-OCR/classifier_tester.1.html

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fb84879ff16408961c96c40dd2640647cf74b8d779a614ef4f86e6eb8daa2b0c
+size 19529

software/Tesseract-OCR/classifier_tester.exe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e059fdeafdc95a6a4220619226ad0d0e7a48e95fd6902c7cdd523af376d3bd74
+size 4987040

software/Tesseract-OCR/cntraining.1.html

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8a2adb8ab0f247b19462b83364bc41dd15004a9d5cade19c678d5be52d9bc756
+size 18156

software/Tesseract-OCR/cntraining.exe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ea0cf4a9e750cb73bafddcc1bd14193e3dc15f2874a5e5ba4191b4662d0560e
+size 4709776

software/Tesseract-OCR/combine_lang_model.1.html

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9daa4a2944dc0d3c38758670bbbb07dccc17cc2176b2b5e571ce4c99fee04654
+size 21055

software/Tesseract-OCR/combine_lang_model.exe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ddd26b40ab4c2633dade1a2678769e9c51075e552be9161d3ce652f52546697e
+size 3503232

software/Tesseract-OCR/combine_tessdata.1.html

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9111e27f1333055506be7d14948b265e031aa0f834b6f68f67ed8c9a12111c60
+size 27130

software/Tesseract-OCR/combine_tessdata.exe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f905d89cd6bd3737fa78369e6a91ce06a3f10641f56a2e8671e0d6c6168f485a
+size 1281096

software/Tesseract-OCR/dawg2wordlist.1.html

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:81566db3708b5a5dd581255eba17a244e51b3b14d9b6cfd82d529c87a0881cad
+size 18336

software/Tesseract-OCR/dawg2wordlist.exe

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ee984cc2c0967473a2dcc361274f3e296c9b3019f8840b85446e5ca166f8a532
+size 575032

software/Tesseract-OCR/doc/AUTHORS

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d69deaae7deeaa053e597906f19482aa56e3de1536851cd20600a09d888c1224
+size 838

software/Tesseract-OCR/doc/LICENSE

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ddf9be5c28fe27dad143a5dc76eea25222ad1dd68934a047064e56ed2fa40c5
+size 11560

software/Tesseract-OCR/doc/README.md

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7918a2fc5dd61b3ea013601f45007604574452ce4934859dc7f43e2e459c019f
+size 8619

software/Tesseract-OCR/libLerc.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ab688c7c5c4afdfee0f519f942759137067a7876a1c1bb825ee575b0a5760b1
+size 761261

software/Tesseract-OCR/libarchive-13.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:98b9b7e9259f23eab6749ee25cca2c0643f09df6b23c58229fd43bf6249195e9
+size 769154

software/Tesseract-OCR/libb2-1.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7fb8418a1172042dab5227fcd466b1bf293737f5b5cf35359e851516227f109c
+size 34372

software/Tesseract-OCR/libbrotlicommon.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:62f363f6aac0d54ce7b115c0412b19e0fe5480f4e6fc1254f663b0ab6868e46f
+size 143397

software/Tesseract-OCR/libbrotlidec.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d6c0f954b6f2db1c1c5fd5904886ae055859a02ca31b6dde5f96f6b0259d23f
+size 60412

software/Tesseract-OCR/libbz2-1.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b79cd3a7102e359ef6324e98966a67939e58ec2b651ae2847cae84a4dfa453f7
+size 100964

software/Tesseract-OCR/libcairo-2.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3a30ed066ebbb5ed6e97da2c119d61a24e4135518c18f8cd8c61299a5182e720
+size 1207571

software/Tesseract-OCR/libcrypto-3-x64.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2dbc1531aa728e1e5d1b11e2927882b8ee2a12bacb2495f04f0d248d5be82cd1
+size 5094016

software/Tesseract-OCR/libcurl-4.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:40986750cf6a9eb36c5fcd2cff5275c4cf45e99794bbbef8c3ff5380ce051bc7
+size 751728

software/Tesseract-OCR/libdatrie-1.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f760aef8a31b7c3670484ca4eabab1ab8e95c14052167e191463f6371c3f0850
+size 34386

software/Tesseract-OCR/libdeflate.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b6ee7b31a23605d8e8b9be5eae782a3ddf7adf2b7ca2c8e0041bdfd4fe4659ee
+size 91035

software/Tesseract-OCR/libexpat-1.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d9ea333175b1e995bbe6a062567f38ff362f1b2f369fa8ebdb0eeed21813bdc6
+size 188909

software/Tesseract-OCR/libffi-8.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1ed51652b3746c1b4bce497b7041e677e5d80c5ab8198efebb8fc6f75329ae8
+size 33708

software/Tesseract-OCR/libfontconfig-1.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d6c70b7e205f24ff96b29a80849b59c84f8701b311434f64f2bf93bd36bdb163
+size 350185

software/Tesseract-OCR/libfreetype-6.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5297f754dfca04abdb0d21705d5afc3c4122aa25c7afbdd87900313cb780c868
+size 776690

software/Tesseract-OCR/libfribidi-0.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:252dec79246c013d8b32e4b1a034626965c55853aec32c5cafcd643b90d018db
+size 150259

software/Tesseract-OCR/libgcc_s_seh-1.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d0cc0027247c22e73cba10ba9e25ea2444a2b4072cb38e45c2cda7374e12a5c2
+size 117427

software/Tesseract-OCR/libgif-7.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a1a179853ae6fad8eb045b8e9f12c55e6fc99929446b02fa345512354cf5e582
+size 41367

software/Tesseract-OCR/libgio-2.0-0.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4ec4f4ddbc2384dd8a0d27c6d559bc7034b390c6bfba1ef89b755c2291cd8ef9
+size 1800402

software/Tesseract-OCR/libglib-2.0-0.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:876cf35b494d95ab012e7220208501882d5f40eb7c3f421b7a92b05ff9e8c3f9
+size 1448224

software/Tesseract-OCR/libgmodule-2.0-0.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41bf2686dd2b8ffb471ded05b876dea3fcb5913212b809da143adc3e9f98e5bd
+size 26783

software/Tesseract-OCR/libgobject-2.0-0.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e5bd93c3eb8db5e86fe8135ffd6eb246d8f9e5c15b8bc1719ca29586ccf1facd
+size 358489

software/Tesseract-OCR/libgraphite2.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f927efecd958f0e54a24ae297f86cd6aed05e7fc20e5da0882050ab9ec464395
+size 152538

software/Tesseract-OCR/libharfbuzz-0.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7ccf58a209c175b4d8bc583e4866e1fb55414effcddebd5f8b2ea5d2feacb0af
+size 1291986

software/Tesseract-OCR/libiconv-2.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:967189adfbc889fde89aafc867f7a1f02731f8592cf6fd5a4ace1929213e2e13
+size 1118202

software/Tesseract-OCR/libicudt74.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90a20195f28e27f65709211c8fa02af5197038f5746692c49b34f7f1bdd98931
+size 30796806

software/Tesseract-OCR/libicuin74.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:69328f8c98dc27ee26bba561cd9b6077a63dcfca88d601ade3bb50143a171d2e
+size 2958241

software/Tesseract-OCR/libicuuc74.dll

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:faea09e5599364842e832ab3f196ad2f166bada760124d10b2acc67c1c06d55c
+size 1839550