Peter Mutwiri commited on
Commit Β·
2334370
1
Parent(s): b9f0345
fixed all errors
Browse files- app/core/event_hub.py +1 -1
- app/main.py +0 -1
- app/mapper.py +7 -8
- app/routers/analytics_stream.py +1 -6
- app/routers/datasources.py +3 -5
- app/schemas/org_schema.py +1 -2
- app/service/llm_service.py +0 -1
- app/tasks/worker.py +0 -1
app/core/event_hub.py
CHANGED
|
@@ -5,7 +5,7 @@ and read recent stream entries without importing `redis` directly.
|
|
| 5 |
"""
|
| 6 |
import json
|
| 7 |
from datetime import datetime
|
| 8 |
-
from typing import Any, Dict
|
| 9 |
|
| 10 |
from app.deps import get_redis
|
| 11 |
|
|
|
|
| 5 |
"""
|
| 6 |
import json
|
| 7 |
from datetime import datetime
|
| 8 |
+
from typing import Any, Dict
|
| 9 |
|
| 10 |
from app.deps import get_redis
|
| 11 |
|
app/main.py
CHANGED
|
@@ -11,7 +11,6 @@ import subprocess
|
|
| 11 |
import asyncio
|
| 12 |
import threading
|
| 13 |
import pathlib
|
| 14 |
-
from datetime import datetime, timedelta
|
| 15 |
|
| 16 |
# βββ Third-Party ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 17 |
from fastapi import FastAPI, Depends, HTTPException, Request, Query, BackgroundTasks,json
|
|
|
|
| 11 |
import asyncio
|
| 12 |
import threading
|
| 13 |
import pathlib
|
|
|
|
| 14 |
|
| 15 |
# βββ Third-Party ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 16 |
from fastapi import FastAPI, Depends, HTTPException, Request, Query, BackgroundTasks,json
|
app/mapper.py
CHANGED
|
@@ -5,8 +5,7 @@ import duckdb
|
|
| 5 |
import pandas as pd
|
| 6 |
import numpy as np
|
| 7 |
from datetime import datetime, timedelta
|
| 8 |
-
from app.db import get_conn, ensure_raw_table,
|
| 9 |
-
from app.utils.detect_industry import _ALIAS, detect_industry
|
| 10 |
# app/mapper.py (add line 1)
|
| 11 |
from app.hybrid_entity_detector import hybrid_detect_entity_type
|
| 12 |
import time
|
|
@@ -107,7 +106,7 @@ def poll_for_entity(org_id: str, source_id: str, timeout: int = 30) -> dict:
|
|
| 107 |
return entity_info
|
| 108 |
|
| 109 |
# 3. Sleep 5 seconds (gives worker time)
|
| 110 |
-
print(
|
| 111 |
time.sleep(5.0)
|
| 112 |
|
| 113 |
# 4. Second attempt (final)
|
|
@@ -119,7 +118,7 @@ def poll_for_entity(org_id: str, source_id: str, timeout: int = 30) -> dict:
|
|
| 119 |
return entity_info
|
| 120 |
|
| 121 |
# 5. Emergency fallback (worker is dead)
|
| 122 |
-
print(
|
| 123 |
return _fallback_detection(org_id, source_id)
|
| 124 |
|
| 125 |
|
|
@@ -139,7 +138,7 @@ def _fallback_detection(org_id: str, source_id: str) -> dict:
|
|
| 139 |
""").fetchall()
|
| 140 |
|
| 141 |
if not rows:
|
| 142 |
-
print(
|
| 143 |
entity_info = {"entity_type": "UNKNOWN", "confidence": 0.0}
|
| 144 |
else:
|
| 145 |
parsed = [json.loads(r[0]) for r in rows if r[0]]
|
|
@@ -185,7 +184,7 @@ def poll_for_industry(org_id: str, source_id: str, timeout: int = 10) -> dict:
|
|
| 185 |
return industry_info
|
| 186 |
|
| 187 |
# 3. Sleep 5 seconds (gives worker time)
|
| 188 |
-
print(
|
| 189 |
time.sleep(5.0)
|
| 190 |
|
| 191 |
# 4. Second attempt (final)
|
|
@@ -197,7 +196,7 @@ def poll_for_industry(org_id: str, source_id: str, timeout: int = 10) -> dict:
|
|
| 197 |
return industry_info
|
| 198 |
|
| 199 |
# 5. Emergency fallback (worker is dead)
|
| 200 |
-
print(
|
| 201 |
return _fallback_industry_detection(org_id, source_id)
|
| 202 |
#fallback industry detection
|
| 203 |
def _fallback_industry_detection(org_id: str, source_id: str) -> dict:
|
|
@@ -218,7 +217,7 @@ def _fallback_industry_detection(org_id: str, source_id: str) -> dict:
|
|
| 218 |
""").fetchall()
|
| 219 |
|
| 220 |
if not rows:
|
| 221 |
-
print(
|
| 222 |
industry_info = {"industry": "UNKNOWN", "confidence": 0.0}
|
| 223 |
else:
|
| 224 |
parsed = [json.loads(r[0]) for r in rows if r[0]]
|
|
|
|
| 5 |
import pandas as pd
|
| 6 |
import numpy as np
|
| 7 |
from datetime import datetime, timedelta
|
| 8 |
+
from app.db import get_conn, ensure_raw_table, transactional_conn,ensure_schema_versions_table
|
|
|
|
| 9 |
# app/mapper.py (add line 1)
|
| 10 |
from app.hybrid_entity_detector import hybrid_detect_entity_type
|
| 11 |
import time
|
|
|
|
| 106 |
return entity_info
|
| 107 |
|
| 108 |
# 3. Sleep 5 seconds (gives worker time)
|
| 109 |
+
print("[poll] π First check failed, sleeping 5s...")
|
| 110 |
time.sleep(5.0)
|
| 111 |
|
| 112 |
# 4. Second attempt (final)
|
|
|
|
| 118 |
return entity_info
|
| 119 |
|
| 120 |
# 5. Emergency fallback (worker is dead)
|
| 121 |
+
print("[poll] β οΈ Both attempts failed - using direct detection")
|
| 122 |
return _fallback_detection(org_id, source_id)
|
| 123 |
|
| 124 |
|
|
|
|
| 138 |
""").fetchall()
|
| 139 |
|
| 140 |
if not rows:
|
| 141 |
+
print("[fallback] β No data found, returning UNKNOWN")
|
| 142 |
entity_info = {"entity_type": "UNKNOWN", "confidence": 0.0}
|
| 143 |
else:
|
| 144 |
parsed = [json.loads(r[0]) for r in rows if r[0]]
|
|
|
|
| 184 |
return industry_info
|
| 185 |
|
| 186 |
# 3. Sleep 5 seconds (gives worker time)
|
| 187 |
+
print("[poll_industry] π First check failed, sleeping 5s...")
|
| 188 |
time.sleep(5.0)
|
| 189 |
|
| 190 |
# 4. Second attempt (final)
|
|
|
|
| 196 |
return industry_info
|
| 197 |
|
| 198 |
# 5. Emergency fallback (worker is dead)
|
| 199 |
+
print("[poll_industry] β οΈ Both attempts failed - using direct detection")
|
| 200 |
return _fallback_industry_detection(org_id, source_id)
|
| 201 |
#fallback industry detection
|
| 202 |
def _fallback_industry_detection(org_id: str, source_id: str) -> dict:
|
|
|
|
| 217 |
""").fetchall()
|
| 218 |
|
| 219 |
if not rows:
|
| 220 |
+
print("[fallback_industry] β No data found, returning UNKNOWN")
|
| 221 |
industry_info = {"industry": "UNKNOWN", "confidence": 0.0}
|
| 222 |
else:
|
| 223 |
parsed = [json.loads(r[0]) for r in rows if r[0]]
|
app/routers/analytics_stream.py
CHANGED
|
@@ -1,14 +1,9 @@
|
|
| 1 |
# app/routers/analytics_stream.py
|
| 2 |
from fastapi import APIRouter, HTTPException, Query,BackgroundTasks, Body, Depends
|
| 3 |
from typing import List, Dict
|
| 4 |
-
import json
|
| 5 |
-
import asyncio
|
| 6 |
from datetime import datetime
|
| 7 |
import logging
|
| 8 |
-
from app.deps import get_current_user
|
| 9 |
-
from app.core.event_hub import event_hub
|
| 10 |
-
import uuid
|
| 11 |
-
from app.qstash_client import publish_message, is_qstash_available
|
| 12 |
from app.core.event_hub import event_hub
|
| 13 |
logger = logging.getLogger(__name__)
|
| 14 |
router = APIRouter(prefix="/api/v1/analytics/stream", tags=["analytics"])
|
|
|
|
| 1 |
# app/routers/analytics_stream.py
|
| 2 |
from fastapi import APIRouter, HTTPException, Query,BackgroundTasks, Body, Depends
|
| 3 |
from typing import List, Dict
|
|
|
|
|
|
|
| 4 |
from datetime import datetime
|
| 5 |
import logging
|
| 6 |
+
from app.deps import get_current_user
|
|
|
|
|
|
|
|
|
|
| 7 |
from app.core.event_hub import event_hub
|
| 8 |
logger = logging.getLogger(__name__)
|
| 9 |
router = APIRouter(prefix="/api/v1/analytics/stream", tags=["analytics"])
|
app/routers/datasources.py
CHANGED
|
@@ -1,16 +1,14 @@
|
|
| 1 |
-
from fastapi import APIRouter, Query,
|
| 2 |
from typing import Dict, Any, List, Union
|
| 3 |
from fastapi.responses import JSONResponse
|
| 4 |
from pydantic import BaseModel
|
| 5 |
-
from typing import List, Any, Dict, Union
|
| 6 |
from app.deps import verify_api_key,get_current_user
|
| 7 |
-
from app.db import
|
| 8 |
from app.mapper import canonify_df
|
| 9 |
from app.routers.socket import sio
|
| 10 |
import pandas as pd
|
| 11 |
import json
|
| 12 |
-
import
|
| 13 |
-
from datetime import datetime, timedelta
|
| 14 |
from app.core.event_hub import event_hub
|
| 15 |
import logging
|
| 16 |
logger = logging.getLogger(__name__)
|
|
|
|
| 1 |
+
from fastapi import APIRouter, Query, Depends, HTTPException
|
| 2 |
from typing import Dict, Any, List, Union
|
| 3 |
from fastapi.responses import JSONResponse
|
| 4 |
from pydantic import BaseModel
|
|
|
|
| 5 |
from app.deps import verify_api_key,get_current_user
|
| 6 |
+
from app.db import bootstrap
|
| 7 |
from app.mapper import canonify_df
|
| 8 |
from app.routers.socket import sio
|
| 9 |
import pandas as pd
|
| 10 |
import json
|
| 11 |
+
from datetime import datetime
|
|
|
|
| 12 |
from app.core.event_hub import event_hub
|
| 13 |
import logging
|
| 14 |
logger = logging.getLogger(__name__)
|
app/schemas/org_schema.py
CHANGED
|
@@ -1,8 +1,7 @@
|
|
| 1 |
# app/schemas/org_schema.py
|
| 2 |
-
from typing import Dict, Optional, List,
|
| 3 |
import json
|
| 4 |
import logging
|
| 5 |
-
from datetime import datetime
|
| 6 |
from app.core.event_hub import event_hub
|
| 7 |
from app.service.llm_service import LocalLLMService
|
| 8 |
from app.service.vector_service import VectorService # Your existing vector service
|
|
|
|
| 1 |
# app/schemas/org_schema.py
|
| 2 |
+
from typing import Dict, Optional, List, Tuple
|
| 3 |
import json
|
| 4 |
import logging
|
|
|
|
| 5 |
from app.core.event_hub import event_hub
|
| 6 |
from app.service.llm_service import LocalLLMService
|
| 7 |
from app.service.vector_service import VectorService # Your existing vector service
|
app/service/llm_service.py
CHANGED
|
@@ -4,7 +4,6 @@ from transformers import AutoModelForCausalLM, AutoTokenizer, pipeline
|
|
| 4 |
from app.deps import HF_API_TOKEN
|
| 5 |
import logging
|
| 6 |
from threading import Thread, Lock
|
| 7 |
-
import time
|
| 8 |
import json
|
| 9 |
import os
|
| 10 |
# redis access not required here; use event_hub if needed
|
|
|
|
| 4 |
from app.deps import HF_API_TOKEN
|
| 5 |
import logging
|
| 6 |
from threading import Thread, Lock
|
|
|
|
| 7 |
import json
|
| 8 |
import os
|
| 9 |
# redis access not required here; use event_hub if needed
|
app/tasks/worker.py
CHANGED
|
@@ -11,7 +11,6 @@ from app.core.event_hub import event_hub
|
|
| 11 |
from app.service.ai_service import ai_service
|
| 12 |
from app.deps import get_duckdb
|
| 13 |
from app.hybrid_entity_detector import hybrid_detect_entity_type, hybrid_detect_industry_type
|
| 14 |
-
from app.utils.detect_industry import detect_industry as rule_based_detect
|
| 15 |
|
| 16 |
# ββ Graceful Shutdown ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 17 |
def shutdown(signum, frame):
|
|
|
|
| 11 |
from app.service.ai_service import ai_service
|
| 12 |
from app.deps import get_duckdb
|
| 13 |
from app.hybrid_entity_detector import hybrid_detect_entity_type, hybrid_detect_industry_type
|
|
|
|
| 14 |
|
| 15 |
# ββ Graceful Shutdown ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ
|
| 16 |
def shutdown(signum, frame):
|