Spaces:
Running
Running
Commit ·
ae0c2d4
1
Parent(s): 1494f68
feat: json schema
Browse files- .gitignore +2 -1
- app/api/v1/chat.py +5 -1
- app/api/v1/convert.py +19 -0
- app/models/schemas.py +2 -0
- app/services/chat_service.py +79 -32
- app/services/extraction_service.py +170 -2
- app/utils/json_utils.py +13 -1
- app/utils/schema_utils.py +156 -0
- pyproject.toml +1 -0
- requirements.txt +3 -0
.gitignore
CHANGED
|
@@ -130,4 +130,5 @@ local_deploy.py
|
|
| 130 |
test_vector_store_async.py
|
| 131 |
deploy_sdk.py
|
| 132 |
tests
|
| 133 |
-
deploy_hf.py
|
|
|
|
|
|
| 130 |
test_vector_store_async.py
|
| 131 |
deploy_sdk.py
|
| 132 |
tests
|
| 133 |
+
deploy_hf.py
|
| 134 |
+
.mimocode
|
app/api/v1/chat.py
CHANGED
|
@@ -10,6 +10,7 @@ from fastapi.responses import StreamingResponse
|
|
| 10 |
|
| 11 |
from app.api.deps import require_auth
|
| 12 |
from app.services.chat_service import _stream_chat_completion, chat_completion
|
|
|
|
| 13 |
|
| 14 |
logger = logging.getLogger(__name__)
|
| 15 |
|
|
@@ -113,7 +114,10 @@ async def create_chat_completion(
|
|
| 113 |
temperature = body.get("temperature", 0.7)
|
| 114 |
top_p = body.get("top_p", 0.9)
|
| 115 |
|
| 116 |
-
|
|
|
|
|
|
|
|
|
|
| 117 |
|
| 118 |
redis = getattr(request.app.state, "redis", None)
|
| 119 |
scripts = getattr(request.app.state, "scripts", None)
|
|
|
|
| 10 |
|
| 11 |
from app.api.deps import require_auth
|
| 12 |
from app.services.chat_service import _stream_chat_completion, chat_completion
|
| 13 |
+
from app.utils.schema_utils import validate_response_format
|
| 14 |
|
| 15 |
logger = logging.getLogger(__name__)
|
| 16 |
|
|
|
|
| 114 |
temperature = body.get("temperature", 0.7)
|
| 115 |
top_p = body.get("top_p", 0.9)
|
| 116 |
|
| 117 |
+
try:
|
| 118 |
+
response_format = validate_response_format(body)
|
| 119 |
+
except ValueError as e:
|
| 120 |
+
raise HTTPException(status_code=400, detail=str(e))
|
| 121 |
|
| 122 |
redis = getattr(request.app.state, "redis", None)
|
| 123 |
scripts = getattr(request.app.state, "scripts", None)
|
app/api/v1/convert.py
CHANGED
|
@@ -63,6 +63,8 @@ async def _build_response(
|
|
| 63 |
filename: Optional[str] = None,
|
| 64 |
raw_data: Optional[bytes] = None,
|
| 65 |
mappings: Optional[Dict[str, Dict[str, Any]]] = None,
|
|
|
|
|
|
|
| 66 |
extraction_service: ExtractionService = None,
|
| 67 |
clean_content: bool = False,
|
| 68 |
text_cleaner_service: TextCleanerService = None,
|
|
@@ -84,6 +86,8 @@ async def _build_response(
|
|
| 84 |
content,
|
| 85 |
mappings,
|
| 86 |
raw_data,
|
|
|
|
|
|
|
| 87 |
)
|
| 88 |
if "error" in json_result:
|
| 89 |
error_message = json_result["error"]
|
|
@@ -146,6 +150,8 @@ async def convert_file(
|
|
| 146 |
return_json: bool = Form(False),
|
| 147 |
clean_content: bool = Query(False),
|
| 148 |
mappings: Optional[str] = Form(None, description="JSON string with field mappings"),
|
|
|
|
|
|
|
| 149 |
token: str = Depends(require_auth),
|
| 150 |
converter_service: ConverterService = Depends(get_converter_service),
|
| 151 |
extraction_service: ExtractionService = Depends(get_extraction_service),
|
|
@@ -161,6 +167,13 @@ async def convert_file(
|
|
| 161 |
except json_mod.JSONDecodeError:
|
| 162 |
raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in mappings parameter."})
|
| 163 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 164 |
_logger.info("Received request to convert file: %s", file.filename)
|
| 165 |
raw = await file.read()
|
| 166 |
if len(raw) > _MAX_UPLOAD_BYTES:
|
|
@@ -188,6 +201,8 @@ async def convert_file(
|
|
| 188 |
filename=file.filename,
|
| 189 |
raw_data=raw,
|
| 190 |
mappings=parsed_mappings,
|
|
|
|
|
|
|
| 191 |
extraction_service=extraction_service,
|
| 192 |
clean_content=clean_content,
|
| 193 |
text_cleaner_service=text_cleaner_service,
|
|
@@ -236,6 +251,8 @@ async def convert_url(
|
|
| 236 |
filename=filename,
|
| 237 |
raw_data=raw_data,
|
| 238 |
mappings=body.mappings,
|
|
|
|
|
|
|
| 239 |
extraction_service=extraction_service,
|
| 240 |
clean_content=clean_content,
|
| 241 |
text_cleaner_service=text_cleaner_service,
|
|
@@ -258,6 +275,8 @@ async def convert_url(
|
|
| 258 |
return_json=body.return_json,
|
| 259 |
filename=filename,
|
| 260 |
mappings=body.mappings,
|
|
|
|
|
|
|
| 261 |
extraction_service=extraction_service,
|
| 262 |
clean_content=clean_content,
|
| 263 |
text_cleaner_service=text_cleaner_service,
|
|
|
|
| 63 |
filename: Optional[str] = None,
|
| 64 |
raw_data: Optional[bytes] = None,
|
| 65 |
mappings: Optional[Dict[str, Dict[str, Any]]] = None,
|
| 66 |
+
json_schema: Optional[Dict[str, Any]] = None,
|
| 67 |
+
schema_name: str = "extraction",
|
| 68 |
extraction_service: ExtractionService = None,
|
| 69 |
clean_content: bool = False,
|
| 70 |
text_cleaner_service: TextCleanerService = None,
|
|
|
|
| 86 |
content,
|
| 87 |
mappings,
|
| 88 |
raw_data,
|
| 89 |
+
json_schema,
|
| 90 |
+
schema_name,
|
| 91 |
)
|
| 92 |
if "error" in json_result:
|
| 93 |
error_message = json_result["error"]
|
|
|
|
| 150 |
return_json: bool = Form(False),
|
| 151 |
clean_content: bool = Query(False),
|
| 152 |
mappings: Optional[str] = Form(None, description="JSON string with field mappings"),
|
| 153 |
+
json_schema: Optional[str] = Form(None, description="JSON Schema for AI-based structured extraction"),
|
| 154 |
+
schema_name: str = Form("extraction", description="Name for the JSON Schema"),
|
| 155 |
token: str = Depends(require_auth),
|
| 156 |
converter_service: ConverterService = Depends(get_converter_service),
|
| 157 |
extraction_service: ExtractionService = Depends(get_extraction_service),
|
|
|
|
| 167 |
except json_mod.JSONDecodeError:
|
| 168 |
raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in mappings parameter."})
|
| 169 |
|
| 170 |
+
parsed_schema = None
|
| 171 |
+
if json_schema:
|
| 172 |
+
try:
|
| 173 |
+
parsed_schema = json_mod.loads(json_schema)
|
| 174 |
+
except json_mod.JSONDecodeError:
|
| 175 |
+
raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in json_schema parameter."})
|
| 176 |
+
|
| 177 |
_logger.info("Received request to convert file: %s", file.filename)
|
| 178 |
raw = await file.read()
|
| 179 |
if len(raw) > _MAX_UPLOAD_BYTES:
|
|
|
|
| 201 |
filename=file.filename,
|
| 202 |
raw_data=raw,
|
| 203 |
mappings=parsed_mappings,
|
| 204 |
+
json_schema=parsed_schema,
|
| 205 |
+
schema_name=schema_name,
|
| 206 |
extraction_service=extraction_service,
|
| 207 |
clean_content=clean_content,
|
| 208 |
text_cleaner_service=text_cleaner_service,
|
|
|
|
| 251 |
filename=filename,
|
| 252 |
raw_data=raw_data,
|
| 253 |
mappings=body.mappings,
|
| 254 |
+
json_schema=body.json_schema,
|
| 255 |
+
schema_name=body.schema_name,
|
| 256 |
extraction_service=extraction_service,
|
| 257 |
clean_content=clean_content,
|
| 258 |
text_cleaner_service=text_cleaner_service,
|
|
|
|
| 275 |
return_json=body.return_json,
|
| 276 |
filename=filename,
|
| 277 |
mappings=body.mappings,
|
| 278 |
+
json_schema=body.json_schema,
|
| 279 |
+
schema_name=body.schema_name,
|
| 280 |
extraction_service=extraction_service,
|
| 281 |
clean_content=clean_content,
|
| 282 |
text_cleaner_service=text_cleaner_service,
|
app/models/schemas.py
CHANGED
|
@@ -70,6 +70,8 @@ class UrlRequest(BaseModel):
|
|
| 70 |
url: str
|
| 71 |
return_json: bool = False
|
| 72 |
mappings: Optional[Dict[str, Dict[str, Any]]] = None
|
|
|
|
|
|
|
| 73 |
|
| 74 |
model_config = {"populate_by_name": True}
|
| 75 |
|
|
|
|
| 70 |
url: str
|
| 71 |
return_json: bool = False
|
| 72 |
mappings: Optional[Dict[str, Dict[str, Any]]] = None
|
| 73 |
+
json_schema: Optional[Dict[str, Any]] = None
|
| 74 |
+
schema_name: str = "extraction"
|
| 75 |
|
| 76 |
model_config = {"populate_by_name": True}
|
| 77 |
|
app/services/chat_service.py
CHANGED
|
@@ -22,7 +22,8 @@ from app.config import (
|
|
| 22 |
OPENROUTER_MIMIKA_MODEL,
|
| 23 |
get_settings,
|
| 24 |
)
|
| 25 |
-
from app.utils.json_utils import extract_json_blocks
|
|
|
|
| 26 |
|
| 27 |
logger = logging.getLogger(__name__)
|
| 28 |
_settings = get_settings()
|
|
@@ -81,33 +82,60 @@ def inject_system_identity(
|
|
| 81 |
|
| 82 |
def prepare_messages(
|
| 83 |
messages: List[Dict[str, str]],
|
| 84 |
-
response_format: Optional[Dict[str,
|
| 85 |
) -> List[Dict[str, str]]:
|
| 86 |
-
if not response_format
|
| 87 |
return messages
|
| 88 |
|
| 89 |
-
|
| 90 |
|
| 91 |
-
if
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 92 |
return [
|
| 93 |
-
{
|
| 94 |
-
|
| 95 |
-
"content": f"{messages[0]['content']}\n\n{DEFAULT_JSON_PROMPT}",
|
| 96 |
-
},
|
| 97 |
-
*messages[1:],
|
| 98 |
]
|
| 99 |
|
| 100 |
-
return
|
| 101 |
-
{"role": "system", "content": DEFAULT_JSON_PROMPT},
|
| 102 |
-
*messages,
|
| 103 |
-
]
|
| 104 |
|
| 105 |
|
| 106 |
def attach_json_content(
|
| 107 |
response_data: Dict[str, Any],
|
| 108 |
-
response_format: Optional[Dict[str,
|
| 109 |
) -> None:
|
| 110 |
-
if not response_format
|
|
|
|
|
|
|
|
|
|
|
|
|
| 111 |
return
|
| 112 |
|
| 113 |
try:
|
|
@@ -115,19 +143,38 @@ def attach_json_content(
|
|
| 115 |
if not choices:
|
| 116 |
return
|
| 117 |
content = choices[0].get("message", {}).get("content", "")
|
| 118 |
-
if content:
|
| 119 |
-
|
| 120 |
-
|
| 121 |
-
|
| 122 |
-
|
| 123 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 124 |
except Exception as exc:
|
| 125 |
response_data["parsed"] = {"error": str(exc)}
|
| 126 |
|
| 127 |
|
| 128 |
async def call_openrouter_mimika(
|
| 129 |
messages: List[Dict[str, str]],
|
| 130 |
-
response_format: Optional[Dict[str,
|
| 131 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 132 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 133 |
top_p: float = DEFAULT_TOP_P,
|
|
@@ -265,7 +312,7 @@ async def call_meganova(
|
|
| 265 |
redis: Redis,
|
| 266 |
scripts: Dict[str, str],
|
| 267 |
messages: List[Dict[str, str]],
|
| 268 |
-
response_format: Optional[Dict[str,
|
| 269 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 270 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 271 |
top_p: float = DEFAULT_TOP_P,
|
|
@@ -362,7 +409,7 @@ async def _get_next_aion_key(redis: Redis) -> Optional[str]:
|
|
| 362 |
async def call_aion_labs(
|
| 363 |
redis: Redis,
|
| 364 |
messages: List[Dict[str, str]],
|
| 365 |
-
response_format: Optional[Dict[str,
|
| 366 |
model: str = AION_LABS_DEFAULT_MODEL,
|
| 367 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 368 |
temperature: float = DEFAULT_TEMPERATURE,
|
|
@@ -510,7 +557,7 @@ async def chat_completion(
|
|
| 510 |
|
| 511 |
async def call_meganova_no_redis(
|
| 512 |
messages: List[Dict[str, str]],
|
| 513 |
-
response_format: Optional[Dict[str,
|
| 514 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 515 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 516 |
top_p: float = DEFAULT_TOP_P,
|
|
@@ -556,7 +603,7 @@ async def call_meganova_no_redis(
|
|
| 556 |
|
| 557 |
async def call_aion_labs_no_redis(
|
| 558 |
messages: List[Dict[str, str]],
|
| 559 |
-
response_format: Optional[Dict[str,
|
| 560 |
model: str = AION_LABS_DEFAULT_MODEL,
|
| 561 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 562 |
temperature: float = DEFAULT_TEMPERATURE,
|
|
@@ -625,7 +672,7 @@ async def _stream_meganova(
|
|
| 625 |
redis: Redis,
|
| 626 |
scripts: Dict[str, str],
|
| 627 |
messages: List[Dict[str, str]],
|
| 628 |
-
response_format: Optional[Dict[str,
|
| 629 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 630 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 631 |
top_p: float = DEFAULT_TOP_P,
|
|
@@ -697,7 +744,7 @@ async def _stream_meganova(
|
|
| 697 |
|
| 698 |
async def _stream_meganova_no_redis(
|
| 699 |
messages: List[Dict[str, str]],
|
| 700 |
-
response_format: Optional[Dict[str,
|
| 701 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 702 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 703 |
top_p: float = DEFAULT_TOP_P,
|
|
@@ -741,7 +788,7 @@ async def _stream_meganova_no_redis(
|
|
| 741 |
async def _stream_aion_labs(
|
| 742 |
redis: Redis,
|
| 743 |
messages: List[Dict[str, str]],
|
| 744 |
-
response_format: Optional[Dict[str,
|
| 745 |
model: str = AION_LABS_DEFAULT_MODEL,
|
| 746 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 747 |
temperature: float = DEFAULT_TEMPERATURE,
|
|
@@ -792,7 +839,7 @@ async def _stream_aion_labs(
|
|
| 792 |
|
| 793 |
async def _stream_aion_labs_no_redis(
|
| 794 |
messages: List[Dict[str, str]],
|
| 795 |
-
response_format: Optional[Dict[str,
|
| 796 |
model: str = AION_LABS_DEFAULT_MODEL,
|
| 797 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 798 |
temperature: float = DEFAULT_TEMPERATURE,
|
|
@@ -836,7 +883,7 @@ async def _stream_aion_labs_no_redis(
|
|
| 836 |
|
| 837 |
async def _stream_openrouter_mimika(
|
| 838 |
messages: List[Dict[str, str]],
|
| 839 |
-
response_format: Optional[Dict[str,
|
| 840 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 841 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 842 |
top_p: float = DEFAULT_TOP_P,
|
|
|
|
| 22 |
OPENROUTER_MIMIKA_MODEL,
|
| 23 |
get_settings,
|
| 24 |
)
|
| 25 |
+
from app.utils.json_utils import extract_json_blocks, extract_single_json
|
| 26 |
+
from app.utils.schema_utils import generate_schema_prompt, validate_against_schema
|
| 27 |
|
| 28 |
logger = logging.getLogger(__name__)
|
| 29 |
_settings = get_settings()
|
|
|
|
| 82 |
|
| 83 |
def prepare_messages(
|
| 84 |
messages: List[Dict[str, str]],
|
| 85 |
+
response_format: Optional[Dict[str, Any]],
|
| 86 |
) -> List[Dict[str, str]]:
|
| 87 |
+
if not response_format:
|
| 88 |
return messages
|
| 89 |
|
| 90 |
+
rf_type = response_format.get("type")
|
| 91 |
|
| 92 |
+
if rf_type == "json_object":
|
| 93 |
+
has_system = messages and messages[0].get("role") == "system"
|
| 94 |
+
if has_system:
|
| 95 |
+
return [
|
| 96 |
+
{
|
| 97 |
+
"role": "system",
|
| 98 |
+
"content": f"{messages[0]['content']}\n\n{DEFAULT_JSON_PROMPT}",
|
| 99 |
+
},
|
| 100 |
+
*messages[1:],
|
| 101 |
+
]
|
| 102 |
+
return [
|
| 103 |
+
{"role": "system", "content": DEFAULT_JSON_PROMPT},
|
| 104 |
+
*messages,
|
| 105 |
+
]
|
| 106 |
+
|
| 107 |
+
if rf_type == "json_schema":
|
| 108 |
+
js = response_format.get("json_schema", {})
|
| 109 |
+
schema_prompt = generate_schema_prompt(
|
| 110 |
+
schema=js.get("schema", {}),
|
| 111 |
+
name=js.get("name", "response"),
|
| 112 |
+
)
|
| 113 |
+
has_system = messages and messages[0].get("role") == "system"
|
| 114 |
+
if has_system:
|
| 115 |
+
return [
|
| 116 |
+
{
|
| 117 |
+
"role": "system",
|
| 118 |
+
"content": f"{messages[0]['content']}\n\n{schema_prompt}",
|
| 119 |
+
},
|
| 120 |
+
*messages[1:],
|
| 121 |
+
]
|
| 122 |
return [
|
| 123 |
+
{"role": "system", "content": schema_prompt},
|
| 124 |
+
*messages,
|
|
|
|
|
|
|
|
|
|
| 125 |
]
|
| 126 |
|
| 127 |
+
return messages
|
|
|
|
|
|
|
|
|
|
| 128 |
|
| 129 |
|
| 130 |
def attach_json_content(
|
| 131 |
response_data: Dict[str, Any],
|
| 132 |
+
response_format: Optional[Dict[str, Any]],
|
| 133 |
) -> None:
|
| 134 |
+
if not response_format:
|
| 135 |
+
return
|
| 136 |
+
|
| 137 |
+
rf_type = response_format.get("type")
|
| 138 |
+
if rf_type not in ("json_object", "json_schema"):
|
| 139 |
return
|
| 140 |
|
| 141 |
try:
|
|
|
|
| 143 |
if not choices:
|
| 144 |
return
|
| 145 |
content = choices[0].get("message", {}).get("content", "")
|
| 146 |
+
if not content:
|
| 147 |
+
return
|
| 148 |
+
|
| 149 |
+
parsed = extract_single_json(content)
|
| 150 |
+
if parsed is None:
|
| 151 |
+
response_data["parsed"] = {"error": "No valid JSON found in response"}
|
| 152 |
+
return
|
| 153 |
+
|
| 154 |
+
if rf_type == "json_schema":
|
| 155 |
+
schema = response_format["json_schema"].get("schema", {})
|
| 156 |
+
is_valid, validated_data, error_msg = validate_against_schema(parsed, schema)
|
| 157 |
+
if is_valid:
|
| 158 |
+
response_data["parsed"] = validated_data
|
| 159 |
+
else:
|
| 160 |
+
response_data["parsed"] = {
|
| 161 |
+
"error": error_msg,
|
| 162 |
+
"raw_output": parsed,
|
| 163 |
+
}
|
| 164 |
+
else:
|
| 165 |
+
# json_object - existing behavior
|
| 166 |
+
if isinstance(parsed, list) and len(parsed) == 1:
|
| 167 |
+
response_data["parsed"] = parsed[0]
|
| 168 |
+
else:
|
| 169 |
+
response_data["parsed"] = parsed
|
| 170 |
+
|
| 171 |
except Exception as exc:
|
| 172 |
response_data["parsed"] = {"error": str(exc)}
|
| 173 |
|
| 174 |
|
| 175 |
async def call_openrouter_mimika(
|
| 176 |
messages: List[Dict[str, str]],
|
| 177 |
+
response_format: Optional[Dict[str, Any]],
|
| 178 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 179 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 180 |
top_p: float = DEFAULT_TOP_P,
|
|
|
|
| 312 |
redis: Redis,
|
| 313 |
scripts: Dict[str, str],
|
| 314 |
messages: List[Dict[str, str]],
|
| 315 |
+
response_format: Optional[Dict[str, Any]],
|
| 316 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 317 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 318 |
top_p: float = DEFAULT_TOP_P,
|
|
|
|
| 409 |
async def call_aion_labs(
|
| 410 |
redis: Redis,
|
| 411 |
messages: List[Dict[str, str]],
|
| 412 |
+
response_format: Optional[Dict[str, Any]],
|
| 413 |
model: str = AION_LABS_DEFAULT_MODEL,
|
| 414 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 415 |
temperature: float = DEFAULT_TEMPERATURE,
|
|
|
|
| 557 |
|
| 558 |
async def call_meganova_no_redis(
|
| 559 |
messages: List[Dict[str, str]],
|
| 560 |
+
response_format: Optional[Dict[str, Any]],
|
| 561 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 562 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 563 |
top_p: float = DEFAULT_TOP_P,
|
|
|
|
| 603 |
|
| 604 |
async def call_aion_labs_no_redis(
|
| 605 |
messages: List[Dict[str, str]],
|
| 606 |
+
response_format: Optional[Dict[str, Any]],
|
| 607 |
model: str = AION_LABS_DEFAULT_MODEL,
|
| 608 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 609 |
temperature: float = DEFAULT_TEMPERATURE,
|
|
|
|
| 672 |
redis: Redis,
|
| 673 |
scripts: Dict[str, str],
|
| 674 |
messages: List[Dict[str, str]],
|
| 675 |
+
response_format: Optional[Dict[str, Any]],
|
| 676 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 677 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 678 |
top_p: float = DEFAULT_TOP_P,
|
|
|
|
| 744 |
|
| 745 |
async def _stream_meganova_no_redis(
|
| 746 |
messages: List[Dict[str, str]],
|
| 747 |
+
response_format: Optional[Dict[str, Any]],
|
| 748 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 749 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 750 |
top_p: float = DEFAULT_TOP_P,
|
|
|
|
| 788 |
async def _stream_aion_labs(
|
| 789 |
redis: Redis,
|
| 790 |
messages: List[Dict[str, str]],
|
| 791 |
+
response_format: Optional[Dict[str, Any]],
|
| 792 |
model: str = AION_LABS_DEFAULT_MODEL,
|
| 793 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 794 |
temperature: float = DEFAULT_TEMPERATURE,
|
|
|
|
| 839 |
|
| 840 |
async def _stream_aion_labs_no_redis(
|
| 841 |
messages: List[Dict[str, str]],
|
| 842 |
+
response_format: Optional[Dict[str, Any]],
|
| 843 |
model: str = AION_LABS_DEFAULT_MODEL,
|
| 844 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 845 |
temperature: float = DEFAULT_TEMPERATURE,
|
|
|
|
| 883 |
|
| 884 |
async def _stream_openrouter_mimika(
|
| 885 |
messages: List[Dict[str, str]],
|
| 886 |
+
response_format: Optional[Dict[str, Any]],
|
| 887 |
max_tokens: int = DEFAULT_MAX_TOKENS,
|
| 888 |
temperature: float = DEFAULT_TEMPERATURE,
|
| 889 |
top_p: float = DEFAULT_TOP_P,
|
app/services/extraction_service.py
CHANGED
|
@@ -17,6 +17,12 @@ from app.core.constants import (
|
|
| 17 |
TABULAR_EXTENSIONS,
|
| 18 |
)
|
| 19 |
from app.core.logger import get_logger
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 20 |
|
| 21 |
_logger = get_logger(__name__)
|
| 22 |
_settings = get_settings()
|
|
@@ -471,12 +477,144 @@ class ExtractionService:
|
|
| 471 |
def __init__(self) -> None:
|
| 472 |
self._spacy_labels = VALID_SPACY_LABELS
|
| 473 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 474 |
def extract_structured(
|
| 475 |
self,
|
| 476 |
filename: Union[str, Path],
|
| 477 |
markdown_text: str,
|
| 478 |
mappings: Optional[Dict[str, Dict[str, Any]]] = None,
|
| 479 |
file_data: Optional[bytes] = None,
|
|
|
|
|
|
|
| 480 |
) -> Dict[str, Any]:
|
| 481 |
ext = Path(filename).suffix.lower()
|
| 482 |
|
|
@@ -486,11 +624,41 @@ class ExtractionService:
|
|
| 486 |
result["extractor"] = "pandas"
|
| 487 |
return result
|
| 488 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 489 |
if not mappings:
|
| 490 |
return {
|
| 491 |
"error": (
|
| 492 |
-
f"Cannot extract JSON from '{ext}' files without field mappings. "
|
| 493 |
-
"Provide a 'mappings' object with field extraction rules."
|
| 494 |
),
|
| 495 |
"file_type": ext,
|
| 496 |
}
|
|
|
|
| 17 |
TABULAR_EXTENSIONS,
|
| 18 |
)
|
| 19 |
from app.core.logger import get_logger
|
| 20 |
+
from app.models.domain import count_tokens
|
| 21 |
+
from app.utils.json_utils import extract_single_json
|
| 22 |
+
from app.utils.schema_utils import (
|
| 23 |
+
generate_schema_prompt,
|
| 24 |
+
validate_against_schema,
|
| 25 |
+
)
|
| 26 |
|
| 27 |
_logger = get_logger(__name__)
|
| 28 |
_settings = get_settings()
|
|
|
|
| 477 |
def __init__(self) -> None:
|
| 478 |
self._spacy_labels = VALID_SPACY_LABELS
|
| 479 |
|
| 480 |
+
async def extract_with_schema(
|
| 481 |
+
self,
|
| 482 |
+
markdown_text: str,
|
| 483 |
+
json_schema: Dict[str, Any],
|
| 484 |
+
schema_name: str = "extraction",
|
| 485 |
+
token_threshold: int = 10000,
|
| 486 |
+
model: str = "agentdeck-1.0",
|
| 487 |
+
provider: Optional[str] = None,
|
| 488 |
+
redis: Any = None,
|
| 489 |
+
scripts: Any = None,
|
| 490 |
+
) -> Dict[str, Any]:
|
| 491 |
+
"""Extract structured data from non-tabular file content using AI + JSON Schema.
|
| 492 |
+
|
| 493 |
+
Validates the schema, counts tokens, generates a schema-guided system prompt,
|
| 494 |
+
sends to AI, parses and validates the response against the schema.
|
| 495 |
+
"""
|
| 496 |
+
from jsonschema import Draft202012Validator, SchemaError, ValidationError
|
| 497 |
+
from app.services.chat_service import chat_completion
|
| 498 |
+
|
| 499 |
+
# 1. Validate the JSON Schema
|
| 500 |
+
try:
|
| 501 |
+
Draft202012Validator.check_schema(json_schema)
|
| 502 |
+
except (ValidationError, SchemaError) as e:
|
| 503 |
+
return {
|
| 504 |
+
"error": f"Invalid JSON Schema: {e.message}",
|
| 505 |
+
"extractor": "ai_schema",
|
| 506 |
+
}
|
| 507 |
+
|
| 508 |
+
# 2. Count tokens using existing token counter
|
| 509 |
+
token_count = count_tokens(markdown_text)
|
| 510 |
+
_logger.info("Schema extraction: %d tokens in content", token_count)
|
| 511 |
+
|
| 512 |
+
if token_count >= token_threshold:
|
| 513 |
+
return {
|
| 514 |
+
"error": (
|
| 515 |
+
f"Content has {token_count} tokens, exceeding threshold of "
|
| 516 |
+
f"{token_threshold}. Content too large for schema extraction."
|
| 517 |
+
),
|
| 518 |
+
"extractor": "ai_schema",
|
| 519 |
+
"token_count": token_count,
|
| 520 |
+
}
|
| 521 |
+
|
| 522 |
+
# 3. Generate schema-guided system prompt
|
| 523 |
+
system_prompt = generate_schema_prompt(schema=json_schema, name=schema_name)
|
| 524 |
+
|
| 525 |
+
# 4. Build messages and send to AI
|
| 526 |
+
messages = [
|
| 527 |
+
{"role": "system", "content": system_prompt},
|
| 528 |
+
{"role": "user", "content": f"Extract structured data from the following content:\n\n{markdown_text}"},
|
| 529 |
+
]
|
| 530 |
+
|
| 531 |
+
response_format = {
|
| 532 |
+
"type": "json_schema",
|
| 533 |
+
"json_schema": {
|
| 534 |
+
"name": schema_name,
|
| 535 |
+
"strict": True,
|
| 536 |
+
"schema": json_schema,
|
| 537 |
+
},
|
| 538 |
+
}
|
| 539 |
+
|
| 540 |
+
try:
|
| 541 |
+
result = await chat_completion(
|
| 542 |
+
messages=messages,
|
| 543 |
+
model=model,
|
| 544 |
+
response_format=response_format,
|
| 545 |
+
max_tokens=12000,
|
| 546 |
+
temperature=0.0,
|
| 547 |
+
top_p=0.9,
|
| 548 |
+
provider=provider,
|
| 549 |
+
redis=redis,
|
| 550 |
+
scripts=scripts,
|
| 551 |
+
)
|
| 552 |
+
except RuntimeError as e:
|
| 553 |
+
return {
|
| 554 |
+
"error": f"AI request failed: {e}",
|
| 555 |
+
"extractor": "ai_schema",
|
| 556 |
+
}
|
| 557 |
+
|
| 558 |
+
# 5. Parse JSON from AI response
|
| 559 |
+
choices = result.get("choices", [])
|
| 560 |
+
if not choices:
|
| 561 |
+
return {
|
| 562 |
+
"error": "No response from AI",
|
| 563 |
+
"extractor": "ai_schema",
|
| 564 |
+
}
|
| 565 |
+
|
| 566 |
+
content = choices[0].get("message", {}).get("content", "")
|
| 567 |
+
if not content:
|
| 568 |
+
return {
|
| 569 |
+
"error": "Empty AI response",
|
| 570 |
+
"extractor": "ai_schema",
|
| 571 |
+
}
|
| 572 |
+
|
| 573 |
+
# Use existing parsed field if available (from attach_json_content)
|
| 574 |
+
parsed_from_result = result.get("parsed")
|
| 575 |
+
if parsed_from_result is not None:
|
| 576 |
+
if not isinstance(parsed_from_result, dict) or "error" not in parsed_from_result:
|
| 577 |
+
parsed_data = parsed_from_result
|
| 578 |
+
else:
|
| 579 |
+
parsed_data = None
|
| 580 |
+
else:
|
| 581 |
+
parsed_data = None
|
| 582 |
+
|
| 583 |
+
if parsed_data is None:
|
| 584 |
+
parsed_data = extract_single_json(content)
|
| 585 |
+
|
| 586 |
+
if parsed_data is None:
|
| 587 |
+
return {
|
| 588 |
+
"error": "Failed to extract JSON from AI response",
|
| 589 |
+
"extractor": "ai_schema",
|
| 590 |
+
"raw_response": content[:500],
|
| 591 |
+
}
|
| 592 |
+
|
| 593 |
+
# 6. Validate against schema
|
| 594 |
+
is_valid, validated_data, error_msg = validate_against_schema(parsed_data, json_schema)
|
| 595 |
+
if not is_valid:
|
| 596 |
+
return {
|
| 597 |
+
"error": f"Schema validation failed: {error_msg}",
|
| 598 |
+
"extractor": "ai_schema",
|
| 599 |
+
"raw_output": parsed_data,
|
| 600 |
+
}
|
| 601 |
+
|
| 602 |
+
return {
|
| 603 |
+
"success": True,
|
| 604 |
+
"extractor": "ai_schema",
|
| 605 |
+
"schema_name": schema_name,
|
| 606 |
+
"token_count": token_count,
|
| 607 |
+
"data": validated_data,
|
| 608 |
+
}
|
| 609 |
+
|
| 610 |
def extract_structured(
|
| 611 |
self,
|
| 612 |
filename: Union[str, Path],
|
| 613 |
markdown_text: str,
|
| 614 |
mappings: Optional[Dict[str, Dict[str, Any]]] = None,
|
| 615 |
file_data: Optional[bytes] = None,
|
| 616 |
+
json_schema: Optional[Dict[str, Any]] = None,
|
| 617 |
+
schema_name: str = "extraction",
|
| 618 |
) -> Dict[str, Any]:
|
| 619 |
ext = Path(filename).suffix.lower()
|
| 620 |
|
|
|
|
| 624 |
result["extractor"] = "pandas"
|
| 625 |
return result
|
| 626 |
|
| 627 |
+
# JSON Schema path: delegate to async AI extraction
|
| 628 |
+
if json_schema:
|
| 629 |
+
import asyncio
|
| 630 |
+
try:
|
| 631 |
+
loop = asyncio.get_running_loop()
|
| 632 |
+
except RuntimeError:
|
| 633 |
+
loop = None
|
| 634 |
+
|
| 635 |
+
if loop and loop.is_running():
|
| 636 |
+
_logger.warning(
|
| 637 |
+
"extract_structured called with json_schema from sync context; "
|
| 638 |
+
"use extract_with_schema() directly for async callers"
|
| 639 |
+
)
|
| 640 |
+
return {
|
| 641 |
+
"error": "JSON Schema extraction requires async context. Use extract_with_schema() instead.",
|
| 642 |
+
"file_type": ext,
|
| 643 |
+
}
|
| 644 |
+
|
| 645 |
+
# Safe to run in a new event loop (CLI / test context)
|
| 646 |
+
result = asyncio.run(
|
| 647 |
+
self.extract_with_schema(
|
| 648 |
+
markdown_text=markdown_text,
|
| 649 |
+
json_schema=json_schema,
|
| 650 |
+
schema_name=schema_name,
|
| 651 |
+
)
|
| 652 |
+
)
|
| 653 |
+
result["file_type"] = ext
|
| 654 |
+
return result
|
| 655 |
+
|
| 656 |
+
# Existing spaCy/regex path
|
| 657 |
if not mappings:
|
| 658 |
return {
|
| 659 |
"error": (
|
| 660 |
+
f"Cannot extract JSON from '{ext}' files without field mappings or json_schema. "
|
| 661 |
+
"Provide a 'mappings' object with field extraction rules, or a 'json_schema' for AI extraction."
|
| 662 |
),
|
| 663 |
"file_type": ext,
|
| 664 |
}
|
app/utils/json_utils.py
CHANGED
|
@@ -3,7 +3,7 @@ from __future__ import annotations
|
|
| 3 |
import json
|
| 4 |
import logging
|
| 5 |
import re
|
| 6 |
-
from typing import Any, List
|
| 7 |
|
| 8 |
logger = logging.getLogger(__name__)
|
| 9 |
|
|
@@ -39,3 +39,15 @@ def extract_json_blocks(text: str) -> List[Any]:
|
|
| 39 |
pass
|
| 40 |
|
| 41 |
return blocks
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 3 |
import json
|
| 4 |
import logging
|
| 5 |
import re
|
| 6 |
+
from typing import Any, List, Optional
|
| 7 |
|
| 8 |
logger = logging.getLogger(__name__)
|
| 9 |
|
|
|
|
| 39 |
pass
|
| 40 |
|
| 41 |
return blocks
|
| 42 |
+
|
| 43 |
+
|
| 44 |
+
def extract_single_json(text: str) -> Optional[Any]:
|
| 45 |
+
"""Extract a single JSON value from text, optimized for structured output.
|
| 46 |
+
|
| 47 |
+
Tries in order:
|
| 48 |
+
1. Parse entire text as JSON
|
| 49 |
+
2. Extract from ```json code fences
|
| 50 |
+
3. Find first {..} or [..] block
|
| 51 |
+
"""
|
| 52 |
+
blocks = extract_json_blocks(text)
|
| 53 |
+
return blocks[0] if blocks else None
|
app/utils/schema_utils.py
ADDED
|
@@ -0,0 +1,156 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from __future__ import annotations
|
| 2 |
+
|
| 3 |
+
import json
|
| 4 |
+
import logging
|
| 5 |
+
from typing import Any, Dict, Optional, Tuple
|
| 6 |
+
|
| 7 |
+
from jsonschema import Draft202012Validator, SchemaError, ValidationError
|
| 8 |
+
|
| 9 |
+
logger = logging.getLogger(__name__)
|
| 10 |
+
|
| 11 |
+
|
| 12 |
+
def validate_response_format(body: Dict[str, Any]) -> Optional[Dict[str, Any]]:
|
| 13 |
+
"""Validate the response_format field from the request body.
|
| 14 |
+
|
| 15 |
+
Accepts:
|
| 16 |
+
- {"type": "json_object"}
|
| 17 |
+
- {"type": "json_schema", "json_schema": {"name": "...", "strict": true, "schema": {...}}}
|
| 18 |
+
|
| 19 |
+
Returns the validated dict or None if response_format is absent.
|
| 20 |
+
Raises ValueError with a human-readable message on invalid input.
|
| 21 |
+
"""
|
| 22 |
+
rf = body.get("response_format")
|
| 23 |
+
if rf is None:
|
| 24 |
+
return None
|
| 25 |
+
|
| 26 |
+
if not isinstance(rf, dict):
|
| 27 |
+
raise ValueError("response_format must be an object")
|
| 28 |
+
|
| 29 |
+
rf_type = rf.get("type")
|
| 30 |
+
if rf_type not in ("json_object", "json_schema"):
|
| 31 |
+
raise ValueError(
|
| 32 |
+
f"response_format.type must be 'json_object' or 'json_schema', got '{rf_type}'"
|
| 33 |
+
)
|
| 34 |
+
|
| 35 |
+
if rf_type == "json_object":
|
| 36 |
+
return rf
|
| 37 |
+
|
| 38 |
+
# json_schema path - validate the nested structure
|
| 39 |
+
js = rf.get("json_schema")
|
| 40 |
+
if not isinstance(js, dict):
|
| 41 |
+
raise ValueError("response_format.json_schema must be an object")
|
| 42 |
+
|
| 43 |
+
name = js.get("name")
|
| 44 |
+
if not isinstance(name, str) or not name.strip():
|
| 45 |
+
raise ValueError("json_schema.name must be a non-empty string")
|
| 46 |
+
|
| 47 |
+
schema = js.get("schema")
|
| 48 |
+
if not isinstance(schema, dict):
|
| 49 |
+
raise ValueError("json_schema.schema must be a JSON Schema object")
|
| 50 |
+
|
| 51 |
+
strict = js.get("strict")
|
| 52 |
+
if strict is not None and not isinstance(strict, bool):
|
| 53 |
+
raise ValueError("json_schema.strict must be a boolean")
|
| 54 |
+
|
| 55 |
+
# Validate the schema itself is a valid JSON Schema
|
| 56 |
+
try:
|
| 57 |
+
Draft202012Validator.check_schema(schema)
|
| 58 |
+
except (ValidationError, SchemaError) as e:
|
| 59 |
+
raise ValueError(f"json_schema.schema is not a valid JSON Schema: {e.message}")
|
| 60 |
+
|
| 61 |
+
return rf
|
| 62 |
+
|
| 63 |
+
|
| 64 |
+
def generate_schema_prompt(
|
| 65 |
+
schema: Dict[str, Any], name: str = "response"
|
| 66 |
+
) -> str:
|
| 67 |
+
"""Generate a system prompt that instructs the LLM to produce output
|
| 68 |
+
conforming to the given JSON Schema."""
|
| 69 |
+
schema_json = json.dumps(schema, indent=2)
|
| 70 |
+
field_instructions = _build_field_instructions(schema)
|
| 71 |
+
|
| 72 |
+
return (
|
| 73 |
+
f'You MUST respond with a single valid JSON object that conforms EXACTLY '
|
| 74 |
+
f'to this JSON Schema named "{name}":\n\n'
|
| 75 |
+
f"```json\n{schema_json}\n```\n\n"
|
| 76 |
+
"CRITICAL RULES:\n"
|
| 77 |
+
"1. Your entire response must be ONLY a JSON object. No text before or after.\n"
|
| 78 |
+
"2. Do NOT wrap the JSON in markdown code fences or any other formatting.\n"
|
| 79 |
+
"3. Every required field MUST be present in your response.\n"
|
| 80 |
+
"4. Use ONLY the types specified in the schema (string, number, integer, boolean, array, object, null).\n"
|
| 81 |
+
"5. Do NOT include any fields that are not defined in the schema properties.\n"
|
| 82 |
+
'6. For "enum" fields, use EXACTLY one of the specified values.\n'
|
| 83 |
+
'7. For "const" fields, use the exact specified value.\n'
|
| 84 |
+
"8. For nested objects, follow the sub-schema recursively.\n\n"
|
| 85 |
+
f"{field_instructions}\n\n"
|
| 86 |
+
"Respond with ONLY the JSON object - no explanation, no markdown, no code fences."
|
| 87 |
+
)
|
| 88 |
+
|
| 89 |
+
|
| 90 |
+
def _build_field_instructions(
|
| 91 |
+
schema: Dict[str, Any], prefix: str = ""
|
| 92 |
+
) -> str:
|
| 93 |
+
"""Recursively build human-readable field instructions from a schema."""
|
| 94 |
+
lines: list[str] = []
|
| 95 |
+
props = schema.get("properties", {})
|
| 96 |
+
required = set(schema.get("required", []))
|
| 97 |
+
|
| 98 |
+
for field_name, field_schema in props.items():
|
| 99 |
+
full_name = f"{prefix}{field_name}" if prefix else field_name
|
| 100 |
+
field_type = field_schema.get("type", "any")
|
| 101 |
+
is_required = field_name in required
|
| 102 |
+
|
| 103 |
+
status = "REQUIRED" if is_required else "optional"
|
| 104 |
+
desc = field_schema.get("description", "")
|
| 105 |
+
|
| 106 |
+
if field_type == "string" and "enum" in field_schema:
|
| 107 |
+
enum_vals = ", ".join(f'"{v}"' for v in field_schema["enum"])
|
| 108 |
+
line = f'- Field "{full_name}" ({status}): Must be one of [{enum_vals}].'
|
| 109 |
+
elif field_type == "string" and "format" in field_schema:
|
| 110 |
+
fmt = field_schema["format"]
|
| 111 |
+
line = f'- Field "{full_name}" ({status}): type=string, format="{fmt}".'
|
| 112 |
+
elif field_type == "array" and "items" in field_schema:
|
| 113 |
+
items = field_schema["items"]
|
| 114 |
+
item_type = items.get("type", "any")
|
| 115 |
+
line = f'- Field "{full_name}" ({status}): type=array of {item_type}.'
|
| 116 |
+
elif field_type == "object" and "properties" in field_schema:
|
| 117 |
+
nested = _build_field_instructions(field_schema, prefix=f"{full_name}.")
|
| 118 |
+
if nested:
|
| 119 |
+
lines.append(f'- Field "{full_name}" ({status}): nested object with fields:')
|
| 120 |
+
lines.append(nested)
|
| 121 |
+
continue
|
| 122 |
+
elif field_type == "null":
|
| 123 |
+
line = f'- Field "{full_name}" ({status}): can be null.'
|
| 124 |
+
else:
|
| 125 |
+
line = f'- Field "{full_name}" ({status}): type={field_type}.'
|
| 126 |
+
|
| 127 |
+
if desc:
|
| 128 |
+
line = line.rstrip(".") + f". {desc}"
|
| 129 |
+
lines.append(line)
|
| 130 |
+
|
| 131 |
+
if not schema.get("additionalProperties", True):
|
| 132 |
+
lines.append("- Do NOT include any additional properties not listed above.")
|
| 133 |
+
|
| 134 |
+
return "\n".join(lines) if lines else ""
|
| 135 |
+
|
| 136 |
+
|
| 137 |
+
def validate_against_schema(
|
| 138 |
+
data: Any, schema: Dict[str, Any]
|
| 139 |
+
) -> Tuple[bool, Any, Optional[str]]:
|
| 140 |
+
"""Validate data against a JSON Schema.
|
| 141 |
+
|
| 142 |
+
Returns:
|
| 143 |
+
(is_valid, data, error_message_or_None)
|
| 144 |
+
"""
|
| 145 |
+
try:
|
| 146 |
+
validator = Draft202012Validator(schema)
|
| 147 |
+
validator.validate(data)
|
| 148 |
+
return True, data, None
|
| 149 |
+
except ValidationError as e:
|
| 150 |
+
path = (
|
| 151 |
+
".".join(str(p) for p in e.absolute_path)
|
| 152 |
+
if e.absolute_path
|
| 153 |
+
else "(root)"
|
| 154 |
+
)
|
| 155 |
+
msg = f"Validation error at '{path}': {e.message}"
|
| 156 |
+
return False, data, msg
|
pyproject.toml
CHANGED
|
@@ -31,6 +31,7 @@ dependencies = [
|
|
| 31 |
"seaborn>=0.13.0",
|
| 32 |
"spacy>=3.7.0",
|
| 33 |
"phonenumbers>=8.13.0",
|
|
|
|
| 34 |
]
|
| 35 |
|
| 36 |
[project.optional-dependencies]
|
|
|
|
| 31 |
"seaborn>=0.13.0",
|
| 32 |
"spacy>=3.7.0",
|
| 33 |
"phonenumbers>=8.13.0",
|
| 34 |
+
"jsonschema>=4.21.0",
|
| 35 |
]
|
| 36 |
|
| 37 |
[project.optional-dependencies]
|
requirements.txt
CHANGED
|
@@ -55,3 +55,6 @@ chardet>=5.2.0
|
|
| 55 |
clean-text>=0.6.0
|
| 56 |
text-unidecode>=1.3
|
| 57 |
Unidecode>=1.3.8
|
|
|
|
|
|
|
|
|
|
|
|
| 55 |
clean-text>=0.6.0
|
| 56 |
text-unidecode>=1.3
|
| 57 |
Unidecode>=1.3.8
|
| 58 |
+
|
| 59 |
+
# JSON Schema validation
|
| 60 |
+
jsonschema>=4.21.0
|