Spaces:
Sleeping
Sleeping
Fix azure sas key for blob container & Fix create weight response
Browse files
externals/databases/pg_crud.py
CHANGED
|
@@ -408,10 +408,10 @@ async def create_filter_and_weight(
|
|
| 408 |
async def create_weight(
|
| 409 |
db: AsyncSession,
|
| 410 |
cv_weight: CVWeight,
|
| 411 |
-
):
|
| 412 |
db.add(cv_weight)
|
| 413 |
-
await db.commit()
|
| 414 |
-
|
| 415 |
|
| 416 |
# =========================
|
| 417 |
# MATCHING
|
|
|
|
| 408 |
async def create_weight(
|
| 409 |
db: AsyncSession,
|
| 410 |
cv_weight: CVWeight,
|
| 411 |
+
) -> CVWeight:
|
| 412 |
db.add(cv_weight)
|
| 413 |
+
await db.commit()
|
| 414 |
+
return cv_weight
|
| 415 |
|
| 416 |
# =========================
|
| 417 |
# MATCHING
|
externals/databases/pg_models.py
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
|
|
| 1 |
from config.get_config import master_config
|
| 2 |
from sqlalchemy import Column, String, Integer, TIMESTAMP, func
|
| 3 |
from sqlalchemy.dialects.postgresql import UUID, ARRAY, DOUBLE_PRECISION, BOOLEAN
|
|
@@ -103,8 +104,8 @@ class CVFilter(Base):
|
|
| 103 |
class CVWeight(Base):
|
| 104 |
__tablename__ = "cv_weight"
|
| 105 |
|
| 106 |
-
weight_id = Column(UUID(as_uuid=True), primary_key=True,
|
| 107 |
-
criteria_id = Column(UUID(as_uuid=True), primary_key=
|
| 108 |
|
| 109 |
gpa_edu_1 = Column(DOUBLE_PRECISION)
|
| 110 |
gpa_edu_2 = Column(DOUBLE_PRECISION)
|
|
|
|
| 1 |
+
import uuid
|
| 2 |
from config.get_config import master_config
|
| 3 |
from sqlalchemy import Column, String, Integer, TIMESTAMP, func
|
| 4 |
from sqlalchemy.dialects.postgresql import UUID, ARRAY, DOUBLE_PRECISION, BOOLEAN
|
|
|
|
| 104 |
class CVWeight(Base):
|
| 105 |
__tablename__ = "cv_weight"
|
| 106 |
|
| 107 |
+
weight_id = Column(UUID(as_uuid=True), primary_key=True, default=uuid.uuid4)
|
| 108 |
+
criteria_id = Column(UUID(as_uuid=True), primary_key=False, nullable=False)
|
| 109 |
|
| 110 |
gpa_edu_1 = Column(DOUBLE_PRECISION)
|
| 111 |
gpa_edu_2 = Column(DOUBLE_PRECISION)
|
externals/storages/azure_blob.py
CHANGED
|
@@ -1,56 +1,70 @@
|
|
| 1 |
-
import os
|
| 2 |
-
import uuid
|
| 3 |
-
from typing import List, Dict, Any
|
| 4 |
-
|
| 5 |
from azure.identity.aio import DefaultAzureCredential
|
| 6 |
from azure.storage.blob.aio import BlobServiceClient, ContainerClient
|
| 7 |
from azure.core.credentials import AzureSasCredential
|
| 8 |
from azure.storage.blob import ContentSettings
|
| 9 |
|
| 10 |
from config.constant import AzureBlobConstants
|
| 11 |
-
from fastapi import UploadFile
|
| 12 |
from utils.logger import get_logger
|
| 13 |
|
| 14 |
logger = get_logger("azure-blob")
|
| 15 |
|
| 16 |
-
# ---------- SINGLETONS ----------
|
| 17 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 18 |
|
| 19 |
|
| 20 |
-
async def
|
| 21 |
try:
|
| 22 |
-
|
| 23 |
-
logger.info(f"✅ Initialized Azure AI Cred: Using **Default Credential**")
|
| 24 |
-
|
| 25 |
-
_blob_service_client = BlobServiceClient(
|
| 26 |
account_url=AzureBlobConstants.ENDPOINT,
|
| 27 |
-
|
|
|
|
| 28 |
)
|
| 29 |
-
return
|
| 30 |
except Exception as E:
|
| 31 |
-
logger.error(f'❌ Getting
|
| 32 |
logger.error(f'⚠️ using DefaultAzureCredential')
|
| 33 |
-
|
| 34 |
-
_blob_service_client = BlobServiceClient(
|
| 35 |
account_url=AzureBlobConstants.ENDPOINT,
|
| 36 |
-
|
|
|
|
| 37 |
)
|
| 38 |
-
return
|
| 39 |
|
| 40 |
|
| 41 |
-
async def get_container_client() -> ContainerClient:
|
| 42 |
-
service = await get_blob_service_client()
|
| 43 |
-
container = service.get_container_client(
|
| 44 |
-
AzureBlobConstants.CONTAINER_NAME
|
| 45 |
-
)
|
| 46 |
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 52 |
|
| 53 |
-
|
| 54 |
|
| 55 |
|
| 56 |
# ---------- HELPERS ----------
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
from azure.identity.aio import DefaultAzureCredential
|
| 2 |
from azure.storage.blob.aio import BlobServiceClient, ContainerClient
|
| 3 |
from azure.core.credentials import AzureSasCredential
|
| 4 |
from azure.storage.blob import ContentSettings
|
| 5 |
|
| 6 |
from config.constant import AzureBlobConstants
|
| 7 |
+
from fastapi import UploadFile
|
| 8 |
from utils.logger import get_logger
|
| 9 |
|
| 10 |
logger = get_logger("azure-blob")
|
| 11 |
|
|
|
|
| 12 |
|
| 13 |
+
# async def get_blob_service_client() -> BlobServiceClient:
|
| 14 |
+
# try:
|
| 15 |
+
# # credential = AzureSasCredential(AzureBlobConstants.SAS_KEY)
|
| 16 |
+
# credential = AzureBlobConstants.SAS_KEY
|
| 17 |
+
# logger.info(f"✅ Initialized Azure AI Cred: Using **Azure SAS Key Credential**")
|
| 18 |
+
|
| 19 |
+
# _blob_service_client = BlobServiceClient(
|
| 20 |
+
# account_url=AzureBlobConstants.ENDPOINT,
|
| 21 |
+
# credential=credential,
|
| 22 |
+
# )
|
| 23 |
+
# return _blob_service_client
|
| 24 |
+
# except Exception as E:
|
| 25 |
+
# logger.error(f'❌ Getting blob service client error, {E}')
|
| 26 |
+
# logger.error(f'⚠️ using DefaultAzureCredential')
|
| 27 |
+
# credential=DefaultAzureCredential()
|
| 28 |
+
# _blob_service_client = BlobServiceClient(
|
| 29 |
+
# account_url=AzureBlobConstants.ENDPOINT,
|
| 30 |
+
# credential=credential,
|
| 31 |
+
# )
|
| 32 |
+
# return _blob_service_client
|
| 33 |
|
| 34 |
|
| 35 |
+
async def get_container_client() -> ContainerClient:
|
| 36 |
try:
|
| 37 |
+
container_client = ContainerClient(
|
|
|
|
|
|
|
|
|
|
| 38 |
account_url=AzureBlobConstants.ENDPOINT,
|
| 39 |
+
container_name=AzureBlobConstants.CONTAINER_NAME,
|
| 40 |
+
credential=AzureSasCredential(AzureBlobConstants.SAS_KEY)
|
| 41 |
)
|
| 42 |
+
return container_client
|
| 43 |
except Exception as E:
|
| 44 |
+
logger.error(f'❌ Getting container client error, {E}')
|
| 45 |
logger.error(f'⚠️ using DefaultAzureCredential')
|
| 46 |
+
container_client = ContainerClient(
|
|
|
|
| 47 |
account_url=AzureBlobConstants.ENDPOINT,
|
| 48 |
+
container_name=AzureBlobConstants.CONTAINER_NAME,
|
| 49 |
+
credential=DefaultAzureCredential()
|
| 50 |
)
|
| 51 |
+
return container_client
|
| 52 |
|
| 53 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 54 |
|
| 55 |
+
# async def get_container_client() -> ContainerClient:
|
| 56 |
+
# service = await get_blob_service_client()
|
| 57 |
+
# container = service.get_container_client(
|
| 58 |
+
# AzureBlobConstants.CONTAINER_NAME
|
| 59 |
+
# )
|
| 60 |
+
|
| 61 |
+
# try:
|
| 62 |
+
# await container.create_container()
|
| 63 |
+
# logger.info("✅ Blob container created")
|
| 64 |
+
# except Exception:
|
| 65 |
+
# pass # already exists
|
| 66 |
|
| 67 |
+
# return container
|
| 68 |
|
| 69 |
|
| 70 |
# ---------- HELPERS ----------
|
interfaces/api/agentic.py
CHANGED
|
@@ -103,6 +103,7 @@ async def create_weight(
|
|
| 103 |
|
| 104 |
cv_weight = CVWeight(
|
| 105 |
criteria_id=criteria_id,
|
|
|
|
| 106 |
|
| 107 |
gpa_edu_1=weight.get("gpa_edu_1"),
|
| 108 |
gpa_edu_2=weight.get("gpa_edu_2"),
|
|
@@ -125,13 +126,12 @@ async def create_weight(
|
|
| 125 |
business_domain=weight.get("business_domain")
|
| 126 |
)
|
| 127 |
|
| 128 |
-
logger.info(f"cv_weight: {cv_weight}")
|
| 129 |
|
| 130 |
data = await agentic_service.weight.create_weight(weight=cv_weight)
|
| 131 |
return {
|
| 132 |
"status": "success",
|
| 133 |
"message": "Weight created successfully",
|
| 134 |
-
"
|
| 135 |
}
|
| 136 |
except Exception as E:
|
| 137 |
logger.error(f"create weight error: {E}")
|
|
|
|
| 103 |
|
| 104 |
cv_weight = CVWeight(
|
| 105 |
criteria_id=criteria_id,
|
| 106 |
+
weight_id=uuid4(),
|
| 107 |
|
| 108 |
gpa_edu_1=weight.get("gpa_edu_1"),
|
| 109 |
gpa_edu_2=weight.get("gpa_edu_2"),
|
|
|
|
| 126 |
business_domain=weight.get("business_domain")
|
| 127 |
)
|
| 128 |
|
|
|
|
| 129 |
|
| 130 |
data = await agentic_service.weight.create_weight(weight=cv_weight)
|
| 131 |
return {
|
| 132 |
"status": "success",
|
| 133 |
"message": "Weight created successfully",
|
| 134 |
+
"data": data
|
| 135 |
}
|
| 136 |
except Exception as E:
|
| 137 |
logger.error(f"create weight error: {E}")
|
services/agentic/weight.py
CHANGED
|
@@ -21,25 +21,27 @@ class AgenticWeightService:
|
|
| 21 |
self.user = user
|
| 22 |
|
| 23 |
|
| 24 |
-
async def create_weight(self, weight: CVWeight) ->
|
| 25 |
"""Return criteria_id:str"""
|
| 26 |
|
| 27 |
try:
|
| 28 |
# check weight existence
|
| 29 |
filter = await get_filter_by_id(self.db, weight.criteria_id)
|
| 30 |
-
|
| 31 |
# loop weight fields is none or empty list
|
| 32 |
weighted_field_name = [] # store field name for not none and not empty list
|
|
|
|
| 33 |
|
| 34 |
for field in filter.__dict__.items():
|
| 35 |
-
if field[1] is None or (isinstance(field[1], list) and not field[1]):
|
| 36 |
-
|
| 37 |
-
|
| 38 |
-
|
| 39 |
|
|
|
|
| 40 |
# check current weight, apply default weight
|
| 41 |
-
new_weight = CVWeight(criteria_id=weight.criteria_id
|
| 42 |
-
|
| 43 |
all_weight = [
|
| 44 |
getattr(weight, field)
|
| 45 |
for field in weighted_field_name
|
|
@@ -67,9 +69,12 @@ class AgenticWeightService:
|
|
| 67 |
setattr(new_weight, field_name, normalized_w)
|
| 68 |
|
| 69 |
# create weight
|
| 70 |
-
await create_weight(self.db, new_weight)
|
| 71 |
-
logger.info(f"Weight created: {
|
| 72 |
-
return
|
|
|
|
|
|
|
|
|
|
| 73 |
except Exception as E:
|
| 74 |
logger.error(f"❌ create weight error: {E}")
|
| 75 |
exc_type, exc_obj, exc_tb = sys.exc_info()
|
|
|
|
| 21 |
self.user = user
|
| 22 |
|
| 23 |
|
| 24 |
+
async def create_weight(self, weight: CVWeight) -> dict:
|
| 25 |
"""Return criteria_id:str"""
|
| 26 |
|
| 27 |
try:
|
| 28 |
# check weight existence
|
| 29 |
filter = await get_filter_by_id(self.db, weight.criteria_id)
|
| 30 |
+
# print("weight.weight_id:", weight.weight_id)
|
| 31 |
# loop weight fields is none or empty list
|
| 32 |
weighted_field_name = [] # store field name for not none and not empty list
|
| 33 |
+
excluded_fields = ["_sa_instance_state","created_at", "criteria_id", "weight_id"]
|
| 34 |
|
| 35 |
for field in filter.__dict__.items():
|
| 36 |
+
# if field[1] is None or (isinstance(field[1], list) and not field[1]):
|
| 37 |
+
# continue
|
| 38 |
+
if field[0] not in excluded_fields:
|
| 39 |
+
weighted_field_name.append(field[0])
|
| 40 |
|
| 41 |
+
print("weighted_field_name:", weighted_field_name)
|
| 42 |
# check current weight, apply default weight
|
| 43 |
+
new_weight = CVWeight(criteria_id=weight.criteria_id,
|
| 44 |
+
weight_id=weight.weight_id)
|
| 45 |
all_weight = [
|
| 46 |
getattr(weight, field)
|
| 47 |
for field in weighted_field_name
|
|
|
|
| 69 |
setattr(new_weight, field_name, normalized_w)
|
| 70 |
|
| 71 |
# create weight
|
| 72 |
+
created_weight = await create_weight(self.db, new_weight)
|
| 73 |
+
logger.info(f"Weight created: {created_weight.weight_id}")
|
| 74 |
+
return {
|
| 75 |
+
"criteria_id": created_weight.criteria_id,
|
| 76 |
+
"weight_id": created_weight.weight_id,
|
| 77 |
+
}
|
| 78 |
except Exception as E:
|
| 79 |
logger.error(f"❌ create weight error: {E}")
|
| 80 |
exc_type, exc_obj, exc_tb = sys.exc_info()
|
services/knowledge/upload_file.py
CHANGED
|
@@ -1,5 +1,5 @@
|
|
| 1 |
from fastapi import UploadFile
|
| 2 |
-
from sqlalchemy.ext.asyncio import AsyncSession
|
| 3 |
from sqlalchemy.exc import IntegrityError
|
| 4 |
from externals.storages.azure_blob import upload_pdf as upload_file
|
| 5 |
from externals.databases.pg_crud import get_file_by_filename, get_file_by_user_id
|
|
|
|
| 1 |
from fastapi import UploadFile
|
| 2 |
+
# from sqlalchemy.ext.asyncio import AsyncSession
|
| 3 |
from sqlalchemy.exc import IntegrityError
|
| 4 |
from externals.storages.azure_blob import upload_pdf as upload_file
|
| 5 |
from externals.databases.pg_crud import get_file_by_filename, get_file_by_user_id
|
services/uploader/azure_blob_service.py
CHANGED
|
@@ -4,68 +4,12 @@ from config.env_constant import EnvFilepath
|
|
| 4 |
dotenv.load_dotenv(EnvFilepath.ENVPATH)
|
| 5 |
|
| 6 |
from config.constant import AzureBlobConstants
|
| 7 |
-
from
|
| 8 |
-
|
| 9 |
-
from azure.core.credentials import AzureSasCredential
|
| 10 |
-
|
| 11 |
-
# from azure.storage.blob import BlobServiceClient, ContainerClient
|
| 12 |
-
from azure.storage.blob.aio import BlobServiceClient, ContainerClient
|
| 13 |
-
# from azure.storage.blob import BlobServiceClient, ContainerClient
|
| 14 |
from fastapi import UploadFile
|
| 15 |
-
|
| 16 |
-
|
| 17 |
from utils.logger import get_logger
|
| 18 |
-
# from src.utils.decorator import trace_runtime
|
| 19 |
-
|
| 20 |
-
logger = get_logger("azure blob")
|
| 21 |
|
| 22 |
|
| 23 |
-
|
| 24 |
-
async def get_blob_service_client() -> BlobServiceClient:
|
| 25 |
-
try:
|
| 26 |
-
credential = AzureSasCredential(AzureBlobConstants.SAS_KEY)
|
| 27 |
-
logger.info(f"✅ Initialized Azure AI Cred: Using **Azure Key Credential**")
|
| 28 |
-
|
| 29 |
-
_blob_service_client = BlobServiceClient(
|
| 30 |
-
account_url=AzureBlobConstants.ENDPOINT,
|
| 31 |
-
credential=credential,
|
| 32 |
-
)
|
| 33 |
-
return _blob_service_client
|
| 34 |
-
except Exception as E:
|
| 35 |
-
logger.error(f'❌ Getting blob service client error, {E}')
|
| 36 |
-
logger.error(f'⚠️ using DefaultAzureCredential')
|
| 37 |
-
credential=DefaultAzureCredential()
|
| 38 |
-
_blob_service_client = BlobServiceClient(
|
| 39 |
-
account_url=AzureBlobConstants.ENDPOINT,
|
| 40 |
-
credential=credential,
|
| 41 |
-
)
|
| 42 |
-
return _blob_service_client
|
| 43 |
-
|
| 44 |
-
# async def get_blob_service_client() -> BlobServiceClient:
|
| 45 |
-
# try:
|
| 46 |
-
# account_url = os.environ.get("azureai__container__endpoint")
|
| 47 |
-
# sas_token = os.environ.get("azureai__search__sas")
|
| 48 |
-
|
| 49 |
-
# if not account_url or not sas_token:
|
| 50 |
-
# raise ValueError("Missing AZURE_STORAGE_ACCOUNT_URL or AZURE_STORAGE_SAS_TOKEN")
|
| 51 |
-
|
| 52 |
-
# blob_service_client = BlobServiceClient(
|
| 53 |
-
# account_url=account_url,
|
| 54 |
-
# credential=sas_token
|
| 55 |
-
# )
|
| 56 |
-
# return blob_service_client
|
| 57 |
-
# except Exception as e:
|
| 58 |
-
# logger.error(f"❌ Getting blob service client error: {e}")
|
| 59 |
-
# raise
|
| 60 |
-
|
| 61 |
-
|
| 62 |
-
async def get_container_client(container_name=os.environ.get("azureai__container__name")) -> ContainerClient:
|
| 63 |
-
try:
|
| 64 |
-
blob_service_client = await get_blob_service_client()
|
| 65 |
-
blob_client = blob_service_client.get_container_client(container_name)
|
| 66 |
-
return blob_client
|
| 67 |
-
except Exception as E:
|
| 68 |
-
logger.error(f'❌ Getting container client error, {E}')
|
| 69 |
|
| 70 |
|
| 71 |
class AzureBlobStorageService:
|
|
@@ -75,12 +19,12 @@ class AzureBlobStorageService:
|
|
| 75 |
"""
|
| 76 |
def __init__(self):
|
| 77 |
# Initialize the BlobServiceClient for the entire application lifetime
|
| 78 |
-
self.blob_service_client = get_blob_service_client()
|
| 79 |
-
self.container_client = get_container_client()
|
| 80 |
self.prefix = AzureBlobConstants.BLOB_PREFIX
|
| 81 |
|
| 82 |
-
if not self.
|
| 83 |
-
|
| 84 |
|
| 85 |
|
| 86 |
async def upload_pdf(self, file: UploadFile) -> str:
|
|
@@ -98,7 +42,8 @@ class AzureBlobStorageService:
|
|
| 98 |
blob_name = file.filename
|
| 99 |
|
| 100 |
# 2. Get the blob client
|
| 101 |
-
|
|
|
|
| 102 |
|
| 103 |
# 3. Read the file content asynchronously
|
| 104 |
# The file is read in chunks to avoid loading the entire file into memory
|
|
|
|
| 4 |
dotenv.load_dotenv(EnvFilepath.ENVPATH)
|
| 5 |
|
| 6 |
from config.constant import AzureBlobConstants
|
| 7 |
+
from externals.storages.azure_blob import get_container_client
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 8 |
from fastapi import UploadFile
|
|
|
|
|
|
|
| 9 |
from utils.logger import get_logger
|
|
|
|
|
|
|
|
|
|
| 10 |
|
| 11 |
|
| 12 |
+
logger = get_logger("azure blob")
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 13 |
|
| 14 |
|
| 15 |
class AzureBlobStorageService:
|
|
|
|
| 19 |
"""
|
| 20 |
def __init__(self):
|
| 21 |
# Initialize the BlobServiceClient for the entire application lifetime
|
| 22 |
+
# self.blob_service_client = get_blob_service_client()
|
| 23 |
+
# self.container_client = get_container_client()
|
| 24 |
self.prefix = AzureBlobConstants.BLOB_PREFIX
|
| 25 |
|
| 26 |
+
# if not self.container_client:
|
| 27 |
+
# raise ValueError("azure container client is not set.")
|
| 28 |
|
| 29 |
|
| 30 |
async def upload_pdf(self, file: UploadFile) -> str:
|
|
|
|
| 42 |
blob_name = file.filename
|
| 43 |
|
| 44 |
# 2. Get the blob client
|
| 45 |
+
container_client = await get_container_client()
|
| 46 |
+
blob_client = container_client.get_blob_client(blob_name)
|
| 47 |
|
| 48 |
# 3. Read the file content asynchronously
|
| 49 |
# The file is read in chunks to avoid loading the entire file into memory
|