light-infer-chat commited on
Commit
ae0c2d4
·
1 Parent(s): 1494f68

feat: json schema

Browse files
.gitignore CHANGED
@@ -130,4 +130,5 @@ local_deploy.py
130
  test_vector_store_async.py
131
  deploy_sdk.py
132
  tests
133
- deploy_hf.py
 
 
130
  test_vector_store_async.py
131
  deploy_sdk.py
132
  tests
133
+ deploy_hf.py
134
+ .mimocode
app/api/v1/chat.py CHANGED
@@ -10,6 +10,7 @@ from fastapi.responses import StreamingResponse
10
 
11
  from app.api.deps import require_auth
12
  from app.services.chat_service import _stream_chat_completion, chat_completion
 
13
 
14
  logger = logging.getLogger(__name__)
15
 
@@ -113,7 +114,10 @@ async def create_chat_completion(
113
  temperature = body.get("temperature", 0.7)
114
  top_p = body.get("top_p", 0.9)
115
 
116
- response_format = body.get("response_format", None)
 
 
 
117
 
118
  redis = getattr(request.app.state, "redis", None)
119
  scripts = getattr(request.app.state, "scripts", None)
 
10
 
11
  from app.api.deps import require_auth
12
  from app.services.chat_service import _stream_chat_completion, chat_completion
13
+ from app.utils.schema_utils import validate_response_format
14
 
15
  logger = logging.getLogger(__name__)
16
 
 
114
  temperature = body.get("temperature", 0.7)
115
  top_p = body.get("top_p", 0.9)
116
 
117
+ try:
118
+ response_format = validate_response_format(body)
119
+ except ValueError as e:
120
+ raise HTTPException(status_code=400, detail=str(e))
121
 
122
  redis = getattr(request.app.state, "redis", None)
123
  scripts = getattr(request.app.state, "scripts", None)
app/api/v1/convert.py CHANGED
@@ -63,6 +63,8 @@ async def _build_response(
63
  filename: Optional[str] = None,
64
  raw_data: Optional[bytes] = None,
65
  mappings: Optional[Dict[str, Dict[str, Any]]] = None,
 
 
66
  extraction_service: ExtractionService = None,
67
  clean_content: bool = False,
68
  text_cleaner_service: TextCleanerService = None,
@@ -84,6 +86,8 @@ async def _build_response(
84
  content,
85
  mappings,
86
  raw_data,
 
 
87
  )
88
  if "error" in json_result:
89
  error_message = json_result["error"]
@@ -146,6 +150,8 @@ async def convert_file(
146
  return_json: bool = Form(False),
147
  clean_content: bool = Query(False),
148
  mappings: Optional[str] = Form(None, description="JSON string with field mappings"),
 
 
149
  token: str = Depends(require_auth),
150
  converter_service: ConverterService = Depends(get_converter_service),
151
  extraction_service: ExtractionService = Depends(get_extraction_service),
@@ -161,6 +167,13 @@ async def convert_file(
161
  except json_mod.JSONDecodeError:
162
  raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in mappings parameter."})
163
 
 
 
 
 
 
 
 
164
  _logger.info("Received request to convert file: %s", file.filename)
165
  raw = await file.read()
166
  if len(raw) > _MAX_UPLOAD_BYTES:
@@ -188,6 +201,8 @@ async def convert_file(
188
  filename=file.filename,
189
  raw_data=raw,
190
  mappings=parsed_mappings,
 
 
191
  extraction_service=extraction_service,
192
  clean_content=clean_content,
193
  text_cleaner_service=text_cleaner_service,
@@ -236,6 +251,8 @@ async def convert_url(
236
  filename=filename,
237
  raw_data=raw_data,
238
  mappings=body.mappings,
 
 
239
  extraction_service=extraction_service,
240
  clean_content=clean_content,
241
  text_cleaner_service=text_cleaner_service,
@@ -258,6 +275,8 @@ async def convert_url(
258
  return_json=body.return_json,
259
  filename=filename,
260
  mappings=body.mappings,
 
 
261
  extraction_service=extraction_service,
262
  clean_content=clean_content,
263
  text_cleaner_service=text_cleaner_service,
 
63
  filename: Optional[str] = None,
64
  raw_data: Optional[bytes] = None,
65
  mappings: Optional[Dict[str, Dict[str, Any]]] = None,
66
+ json_schema: Optional[Dict[str, Any]] = None,
67
+ schema_name: str = "extraction",
68
  extraction_service: ExtractionService = None,
69
  clean_content: bool = False,
70
  text_cleaner_service: TextCleanerService = None,
 
86
  content,
87
  mappings,
88
  raw_data,
89
+ json_schema,
90
+ schema_name,
91
  )
92
  if "error" in json_result:
93
  error_message = json_result["error"]
 
150
  return_json: bool = Form(False),
151
  clean_content: bool = Query(False),
152
  mappings: Optional[str] = Form(None, description="JSON string with field mappings"),
153
+ json_schema: Optional[str] = Form(None, description="JSON Schema for AI-based structured extraction"),
154
+ schema_name: str = Form("extraction", description="Name for the JSON Schema"),
155
  token: str = Depends(require_auth),
156
  converter_service: ConverterService = Depends(get_converter_service),
157
  extraction_service: ExtractionService = Depends(get_extraction_service),
 
167
  except json_mod.JSONDecodeError:
168
  raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in mappings parameter."})
169
 
170
+ parsed_schema = None
171
+ if json_schema:
172
+ try:
173
+ parsed_schema = json_mod.loads(json_schema)
174
+ except json_mod.JSONDecodeError:
175
+ raise HTTPException(status_code=400, detail={"success": False, "message": "Invalid JSON in json_schema parameter."})
176
+
177
  _logger.info("Received request to convert file: %s", file.filename)
178
  raw = await file.read()
179
  if len(raw) > _MAX_UPLOAD_BYTES:
 
201
  filename=file.filename,
202
  raw_data=raw,
203
  mappings=parsed_mappings,
204
+ json_schema=parsed_schema,
205
+ schema_name=schema_name,
206
  extraction_service=extraction_service,
207
  clean_content=clean_content,
208
  text_cleaner_service=text_cleaner_service,
 
251
  filename=filename,
252
  raw_data=raw_data,
253
  mappings=body.mappings,
254
+ json_schema=body.json_schema,
255
+ schema_name=body.schema_name,
256
  extraction_service=extraction_service,
257
  clean_content=clean_content,
258
  text_cleaner_service=text_cleaner_service,
 
275
  return_json=body.return_json,
276
  filename=filename,
277
  mappings=body.mappings,
278
+ json_schema=body.json_schema,
279
+ schema_name=body.schema_name,
280
  extraction_service=extraction_service,
281
  clean_content=clean_content,
282
  text_cleaner_service=text_cleaner_service,
app/models/schemas.py CHANGED
@@ -70,6 +70,8 @@ class UrlRequest(BaseModel):
70
  url: str
71
  return_json: bool = False
72
  mappings: Optional[Dict[str, Dict[str, Any]]] = None
 
 
73
 
74
  model_config = {"populate_by_name": True}
75
 
 
70
  url: str
71
  return_json: bool = False
72
  mappings: Optional[Dict[str, Dict[str, Any]]] = None
73
+ json_schema: Optional[Dict[str, Any]] = None
74
+ schema_name: str = "extraction"
75
 
76
  model_config = {"populate_by_name": True}
77
 
app/services/chat_service.py CHANGED
@@ -22,7 +22,8 @@ from app.config import (
22
  OPENROUTER_MIMIKA_MODEL,
23
  get_settings,
24
  )
25
- from app.utils.json_utils import extract_json_blocks
 
26
 
27
  logger = logging.getLogger(__name__)
28
  _settings = get_settings()
@@ -81,33 +82,60 @@ def inject_system_identity(
81
 
82
  def prepare_messages(
83
  messages: List[Dict[str, str]],
84
- response_format: Optional[Dict[str, str]],
85
  ) -> List[Dict[str, str]]:
86
- if not response_format or response_format.get("type") != "json_object":
87
  return messages
88
 
89
- has_system = messages and messages[0].get("role") == "system"
90
 
91
- if has_system:
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
92
  return [
93
- {
94
- "role": "system",
95
- "content": f"{messages[0]['content']}\n\n{DEFAULT_JSON_PROMPT}",
96
- },
97
- *messages[1:],
98
  ]
99
 
100
- return [
101
- {"role": "system", "content": DEFAULT_JSON_PROMPT},
102
- *messages,
103
- ]
104
 
105
 
106
  def attach_json_content(
107
  response_data: Dict[str, Any],
108
- response_format: Optional[Dict[str, str]],
109
  ) -> None:
110
- if not response_format or response_format.get("type") != "json_object":
 
 
 
 
111
  return
112
 
113
  try:
@@ -115,19 +143,38 @@ def attach_json_content(
115
  if not choices:
116
  return
117
  content = choices[0].get("message", {}).get("content", "")
118
- if content:
119
- parsed = extract_json_blocks(content)
120
- if parsed:
121
- response_data["parsed"] = (
122
- parsed[0] if len(parsed) == 1 else parsed
123
- )
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
124
  except Exception as exc:
125
  response_data["parsed"] = {"error": str(exc)}
126
 
127
 
128
  async def call_openrouter_mimika(
129
  messages: List[Dict[str, str]],
130
- response_format: Optional[Dict[str, str]],
131
  max_tokens: int = DEFAULT_MAX_TOKENS,
132
  temperature: float = DEFAULT_TEMPERATURE,
133
  top_p: float = DEFAULT_TOP_P,
@@ -265,7 +312,7 @@ async def call_meganova(
265
  redis: Redis,
266
  scripts: Dict[str, str],
267
  messages: List[Dict[str, str]],
268
- response_format: Optional[Dict[str, str]],
269
  max_tokens: int = DEFAULT_MAX_TOKENS,
270
  temperature: float = DEFAULT_TEMPERATURE,
271
  top_p: float = DEFAULT_TOP_P,
@@ -362,7 +409,7 @@ async def _get_next_aion_key(redis: Redis) -> Optional[str]:
362
  async def call_aion_labs(
363
  redis: Redis,
364
  messages: List[Dict[str, str]],
365
- response_format: Optional[Dict[str, str]],
366
  model: str = AION_LABS_DEFAULT_MODEL,
367
  max_tokens: int = DEFAULT_MAX_TOKENS,
368
  temperature: float = DEFAULT_TEMPERATURE,
@@ -510,7 +557,7 @@ async def chat_completion(
510
 
511
  async def call_meganova_no_redis(
512
  messages: List[Dict[str, str]],
513
- response_format: Optional[Dict[str, str]],
514
  max_tokens: int = DEFAULT_MAX_TOKENS,
515
  temperature: float = DEFAULT_TEMPERATURE,
516
  top_p: float = DEFAULT_TOP_P,
@@ -556,7 +603,7 @@ async def call_meganova_no_redis(
556
 
557
  async def call_aion_labs_no_redis(
558
  messages: List[Dict[str, str]],
559
- response_format: Optional[Dict[str, str]],
560
  model: str = AION_LABS_DEFAULT_MODEL,
561
  max_tokens: int = DEFAULT_MAX_TOKENS,
562
  temperature: float = DEFAULT_TEMPERATURE,
@@ -625,7 +672,7 @@ async def _stream_meganova(
625
  redis: Redis,
626
  scripts: Dict[str, str],
627
  messages: List[Dict[str, str]],
628
- response_format: Optional[Dict[str, str]],
629
  max_tokens: int = DEFAULT_MAX_TOKENS,
630
  temperature: float = DEFAULT_TEMPERATURE,
631
  top_p: float = DEFAULT_TOP_P,
@@ -697,7 +744,7 @@ async def _stream_meganova(
697
 
698
  async def _stream_meganova_no_redis(
699
  messages: List[Dict[str, str]],
700
- response_format: Optional[Dict[str, str]],
701
  max_tokens: int = DEFAULT_MAX_TOKENS,
702
  temperature: float = DEFAULT_TEMPERATURE,
703
  top_p: float = DEFAULT_TOP_P,
@@ -741,7 +788,7 @@ async def _stream_meganova_no_redis(
741
  async def _stream_aion_labs(
742
  redis: Redis,
743
  messages: List[Dict[str, str]],
744
- response_format: Optional[Dict[str, str]],
745
  model: str = AION_LABS_DEFAULT_MODEL,
746
  max_tokens: int = DEFAULT_MAX_TOKENS,
747
  temperature: float = DEFAULT_TEMPERATURE,
@@ -792,7 +839,7 @@ async def _stream_aion_labs(
792
 
793
  async def _stream_aion_labs_no_redis(
794
  messages: List[Dict[str, str]],
795
- response_format: Optional[Dict[str, str]],
796
  model: str = AION_LABS_DEFAULT_MODEL,
797
  max_tokens: int = DEFAULT_MAX_TOKENS,
798
  temperature: float = DEFAULT_TEMPERATURE,
@@ -836,7 +883,7 @@ async def _stream_aion_labs_no_redis(
836
 
837
  async def _stream_openrouter_mimika(
838
  messages: List[Dict[str, str]],
839
- response_format: Optional[Dict[str, str]],
840
  max_tokens: int = DEFAULT_MAX_TOKENS,
841
  temperature: float = DEFAULT_TEMPERATURE,
842
  top_p: float = DEFAULT_TOP_P,
 
22
  OPENROUTER_MIMIKA_MODEL,
23
  get_settings,
24
  )
25
+ from app.utils.json_utils import extract_json_blocks, extract_single_json
26
+ from app.utils.schema_utils import generate_schema_prompt, validate_against_schema
27
 
28
  logger = logging.getLogger(__name__)
29
  _settings = get_settings()
 
82
 
83
  def prepare_messages(
84
  messages: List[Dict[str, str]],
85
+ response_format: Optional[Dict[str, Any]],
86
  ) -> List[Dict[str, str]]:
87
+ if not response_format:
88
  return messages
89
 
90
+ rf_type = response_format.get("type")
91
 
92
+ if rf_type == "json_object":
93
+ has_system = messages and messages[0].get("role") == "system"
94
+ if has_system:
95
+ return [
96
+ {
97
+ "role": "system",
98
+ "content": f"{messages[0]['content']}\n\n{DEFAULT_JSON_PROMPT}",
99
+ },
100
+ *messages[1:],
101
+ ]
102
+ return [
103
+ {"role": "system", "content": DEFAULT_JSON_PROMPT},
104
+ *messages,
105
+ ]
106
+
107
+ if rf_type == "json_schema":
108
+ js = response_format.get("json_schema", {})
109
+ schema_prompt = generate_schema_prompt(
110
+ schema=js.get("schema", {}),
111
+ name=js.get("name", "response"),
112
+ )
113
+ has_system = messages and messages[0].get("role") == "system"
114
+ if has_system:
115
+ return [
116
+ {
117
+ "role": "system",
118
+ "content": f"{messages[0]['content']}\n\n{schema_prompt}",
119
+ },
120
+ *messages[1:],
121
+ ]
122
  return [
123
+ {"role": "system", "content": schema_prompt},
124
+ *messages,
 
 
 
125
  ]
126
 
127
+ return messages
 
 
 
128
 
129
 
130
  def attach_json_content(
131
  response_data: Dict[str, Any],
132
+ response_format: Optional[Dict[str, Any]],
133
  ) -> None:
134
+ if not response_format:
135
+ return
136
+
137
+ rf_type = response_format.get("type")
138
+ if rf_type not in ("json_object", "json_schema"):
139
  return
140
 
141
  try:
 
143
  if not choices:
144
  return
145
  content = choices[0].get("message", {}).get("content", "")
146
+ if not content:
147
+ return
148
+
149
+ parsed = extract_single_json(content)
150
+ if parsed is None:
151
+ response_data["parsed"] = {"error": "No valid JSON found in response"}
152
+ return
153
+
154
+ if rf_type == "json_schema":
155
+ schema = response_format["json_schema"].get("schema", {})
156
+ is_valid, validated_data, error_msg = validate_against_schema(parsed, schema)
157
+ if is_valid:
158
+ response_data["parsed"] = validated_data
159
+ else:
160
+ response_data["parsed"] = {
161
+ "error": error_msg,
162
+ "raw_output": parsed,
163
+ }
164
+ else:
165
+ # json_object - existing behavior
166
+ if isinstance(parsed, list) and len(parsed) == 1:
167
+ response_data["parsed"] = parsed[0]
168
+ else:
169
+ response_data["parsed"] = parsed
170
+
171
  except Exception as exc:
172
  response_data["parsed"] = {"error": str(exc)}
173
 
174
 
175
  async def call_openrouter_mimika(
176
  messages: List[Dict[str, str]],
177
+ response_format: Optional[Dict[str, Any]],
178
  max_tokens: int = DEFAULT_MAX_TOKENS,
179
  temperature: float = DEFAULT_TEMPERATURE,
180
  top_p: float = DEFAULT_TOP_P,
 
312
  redis: Redis,
313
  scripts: Dict[str, str],
314
  messages: List[Dict[str, str]],
315
+ response_format: Optional[Dict[str, Any]],
316
  max_tokens: int = DEFAULT_MAX_TOKENS,
317
  temperature: float = DEFAULT_TEMPERATURE,
318
  top_p: float = DEFAULT_TOP_P,
 
409
  async def call_aion_labs(
410
  redis: Redis,
411
  messages: List[Dict[str, str]],
412
+ response_format: Optional[Dict[str, Any]],
413
  model: str = AION_LABS_DEFAULT_MODEL,
414
  max_tokens: int = DEFAULT_MAX_TOKENS,
415
  temperature: float = DEFAULT_TEMPERATURE,
 
557
 
558
  async def call_meganova_no_redis(
559
  messages: List[Dict[str, str]],
560
+ response_format: Optional[Dict[str, Any]],
561
  max_tokens: int = DEFAULT_MAX_TOKENS,
562
  temperature: float = DEFAULT_TEMPERATURE,
563
  top_p: float = DEFAULT_TOP_P,
 
603
 
604
  async def call_aion_labs_no_redis(
605
  messages: List[Dict[str, str]],
606
+ response_format: Optional[Dict[str, Any]],
607
  model: str = AION_LABS_DEFAULT_MODEL,
608
  max_tokens: int = DEFAULT_MAX_TOKENS,
609
  temperature: float = DEFAULT_TEMPERATURE,
 
672
  redis: Redis,
673
  scripts: Dict[str, str],
674
  messages: List[Dict[str, str]],
675
+ response_format: Optional[Dict[str, Any]],
676
  max_tokens: int = DEFAULT_MAX_TOKENS,
677
  temperature: float = DEFAULT_TEMPERATURE,
678
  top_p: float = DEFAULT_TOP_P,
 
744
 
745
  async def _stream_meganova_no_redis(
746
  messages: List[Dict[str, str]],
747
+ response_format: Optional[Dict[str, Any]],
748
  max_tokens: int = DEFAULT_MAX_TOKENS,
749
  temperature: float = DEFAULT_TEMPERATURE,
750
  top_p: float = DEFAULT_TOP_P,
 
788
  async def _stream_aion_labs(
789
  redis: Redis,
790
  messages: List[Dict[str, str]],
791
+ response_format: Optional[Dict[str, Any]],
792
  model: str = AION_LABS_DEFAULT_MODEL,
793
  max_tokens: int = DEFAULT_MAX_TOKENS,
794
  temperature: float = DEFAULT_TEMPERATURE,
 
839
 
840
  async def _stream_aion_labs_no_redis(
841
  messages: List[Dict[str, str]],
842
+ response_format: Optional[Dict[str, Any]],
843
  model: str = AION_LABS_DEFAULT_MODEL,
844
  max_tokens: int = DEFAULT_MAX_TOKENS,
845
  temperature: float = DEFAULT_TEMPERATURE,
 
883
 
884
  async def _stream_openrouter_mimika(
885
  messages: List[Dict[str, str]],
886
+ response_format: Optional[Dict[str, Any]],
887
  max_tokens: int = DEFAULT_MAX_TOKENS,
888
  temperature: float = DEFAULT_TEMPERATURE,
889
  top_p: float = DEFAULT_TOP_P,
app/services/extraction_service.py CHANGED
@@ -17,6 +17,12 @@ from app.core.constants import (
17
  TABULAR_EXTENSIONS,
18
  )
19
  from app.core.logger import get_logger
 
 
 
 
 
 
20
 
21
  _logger = get_logger(__name__)
22
  _settings = get_settings()
@@ -471,12 +477,144 @@ class ExtractionService:
471
  def __init__(self) -> None:
472
  self._spacy_labels = VALID_SPACY_LABELS
473
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
474
  def extract_structured(
475
  self,
476
  filename: Union[str, Path],
477
  markdown_text: str,
478
  mappings: Optional[Dict[str, Dict[str, Any]]] = None,
479
  file_data: Optional[bytes] = None,
 
 
480
  ) -> Dict[str, Any]:
481
  ext = Path(filename).suffix.lower()
482
 
@@ -486,11 +624,41 @@ class ExtractionService:
486
  result["extractor"] = "pandas"
487
  return result
488
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
489
  if not mappings:
490
  return {
491
  "error": (
492
- f"Cannot extract JSON from '{ext}' files without field mappings. "
493
- "Provide a 'mappings' object with field extraction rules."
494
  ),
495
  "file_type": ext,
496
  }
 
17
  TABULAR_EXTENSIONS,
18
  )
19
  from app.core.logger import get_logger
20
+ from app.models.domain import count_tokens
21
+ from app.utils.json_utils import extract_single_json
22
+ from app.utils.schema_utils import (
23
+ generate_schema_prompt,
24
+ validate_against_schema,
25
+ )
26
 
27
  _logger = get_logger(__name__)
28
  _settings = get_settings()
 
477
  def __init__(self) -> None:
478
  self._spacy_labels = VALID_SPACY_LABELS
479
 
480
+ async def extract_with_schema(
481
+ self,
482
+ markdown_text: str,
483
+ json_schema: Dict[str, Any],
484
+ schema_name: str = "extraction",
485
+ token_threshold: int = 10000,
486
+ model: str = "agentdeck-1.0",
487
+ provider: Optional[str] = None,
488
+ redis: Any = None,
489
+ scripts: Any = None,
490
+ ) -> Dict[str, Any]:
491
+ """Extract structured data from non-tabular file content using AI + JSON Schema.
492
+
493
+ Validates the schema, counts tokens, generates a schema-guided system prompt,
494
+ sends to AI, parses and validates the response against the schema.
495
+ """
496
+ from jsonschema import Draft202012Validator, SchemaError, ValidationError
497
+ from app.services.chat_service import chat_completion
498
+
499
+ # 1. Validate the JSON Schema
500
+ try:
501
+ Draft202012Validator.check_schema(json_schema)
502
+ except (ValidationError, SchemaError) as e:
503
+ return {
504
+ "error": f"Invalid JSON Schema: {e.message}",
505
+ "extractor": "ai_schema",
506
+ }
507
+
508
+ # 2. Count tokens using existing token counter
509
+ token_count = count_tokens(markdown_text)
510
+ _logger.info("Schema extraction: %d tokens in content", token_count)
511
+
512
+ if token_count >= token_threshold:
513
+ return {
514
+ "error": (
515
+ f"Content has {token_count} tokens, exceeding threshold of "
516
+ f"{token_threshold}. Content too large for schema extraction."
517
+ ),
518
+ "extractor": "ai_schema",
519
+ "token_count": token_count,
520
+ }
521
+
522
+ # 3. Generate schema-guided system prompt
523
+ system_prompt = generate_schema_prompt(schema=json_schema, name=schema_name)
524
+
525
+ # 4. Build messages and send to AI
526
+ messages = [
527
+ {"role": "system", "content": system_prompt},
528
+ {"role": "user", "content": f"Extract structured data from the following content:\n\n{markdown_text}"},
529
+ ]
530
+
531
+ response_format = {
532
+ "type": "json_schema",
533
+ "json_schema": {
534
+ "name": schema_name,
535
+ "strict": True,
536
+ "schema": json_schema,
537
+ },
538
+ }
539
+
540
+ try:
541
+ result = await chat_completion(
542
+ messages=messages,
543
+ model=model,
544
+ response_format=response_format,
545
+ max_tokens=12000,
546
+ temperature=0.0,
547
+ top_p=0.9,
548
+ provider=provider,
549
+ redis=redis,
550
+ scripts=scripts,
551
+ )
552
+ except RuntimeError as e:
553
+ return {
554
+ "error": f"AI request failed: {e}",
555
+ "extractor": "ai_schema",
556
+ }
557
+
558
+ # 5. Parse JSON from AI response
559
+ choices = result.get("choices", [])
560
+ if not choices:
561
+ return {
562
+ "error": "No response from AI",
563
+ "extractor": "ai_schema",
564
+ }
565
+
566
+ content = choices[0].get("message", {}).get("content", "")
567
+ if not content:
568
+ return {
569
+ "error": "Empty AI response",
570
+ "extractor": "ai_schema",
571
+ }
572
+
573
+ # Use existing parsed field if available (from attach_json_content)
574
+ parsed_from_result = result.get("parsed")
575
+ if parsed_from_result is not None:
576
+ if not isinstance(parsed_from_result, dict) or "error" not in parsed_from_result:
577
+ parsed_data = parsed_from_result
578
+ else:
579
+ parsed_data = None
580
+ else:
581
+ parsed_data = None
582
+
583
+ if parsed_data is None:
584
+ parsed_data = extract_single_json(content)
585
+
586
+ if parsed_data is None:
587
+ return {
588
+ "error": "Failed to extract JSON from AI response",
589
+ "extractor": "ai_schema",
590
+ "raw_response": content[:500],
591
+ }
592
+
593
+ # 6. Validate against schema
594
+ is_valid, validated_data, error_msg = validate_against_schema(parsed_data, json_schema)
595
+ if not is_valid:
596
+ return {
597
+ "error": f"Schema validation failed: {error_msg}",
598
+ "extractor": "ai_schema",
599
+ "raw_output": parsed_data,
600
+ }
601
+
602
+ return {
603
+ "success": True,
604
+ "extractor": "ai_schema",
605
+ "schema_name": schema_name,
606
+ "token_count": token_count,
607
+ "data": validated_data,
608
+ }
609
+
610
  def extract_structured(
611
  self,
612
  filename: Union[str, Path],
613
  markdown_text: str,
614
  mappings: Optional[Dict[str, Dict[str, Any]]] = None,
615
  file_data: Optional[bytes] = None,
616
+ json_schema: Optional[Dict[str, Any]] = None,
617
+ schema_name: str = "extraction",
618
  ) -> Dict[str, Any]:
619
  ext = Path(filename).suffix.lower()
620
 
 
624
  result["extractor"] = "pandas"
625
  return result
626
 
627
+ # JSON Schema path: delegate to async AI extraction
628
+ if json_schema:
629
+ import asyncio
630
+ try:
631
+ loop = asyncio.get_running_loop()
632
+ except RuntimeError:
633
+ loop = None
634
+
635
+ if loop and loop.is_running():
636
+ _logger.warning(
637
+ "extract_structured called with json_schema from sync context; "
638
+ "use extract_with_schema() directly for async callers"
639
+ )
640
+ return {
641
+ "error": "JSON Schema extraction requires async context. Use extract_with_schema() instead.",
642
+ "file_type": ext,
643
+ }
644
+
645
+ # Safe to run in a new event loop (CLI / test context)
646
+ result = asyncio.run(
647
+ self.extract_with_schema(
648
+ markdown_text=markdown_text,
649
+ json_schema=json_schema,
650
+ schema_name=schema_name,
651
+ )
652
+ )
653
+ result["file_type"] = ext
654
+ return result
655
+
656
+ # Existing spaCy/regex path
657
  if not mappings:
658
  return {
659
  "error": (
660
+ f"Cannot extract JSON from '{ext}' files without field mappings or json_schema. "
661
+ "Provide a 'mappings' object with field extraction rules, or a 'json_schema' for AI extraction."
662
  ),
663
  "file_type": ext,
664
  }
app/utils/json_utils.py CHANGED
@@ -3,7 +3,7 @@ from __future__ import annotations
3
  import json
4
  import logging
5
  import re
6
- from typing import Any, List
7
 
8
  logger = logging.getLogger(__name__)
9
 
@@ -39,3 +39,15 @@ def extract_json_blocks(text: str) -> List[Any]:
39
  pass
40
 
41
  return blocks
 
 
 
 
 
 
 
 
 
 
 
 
 
3
  import json
4
  import logging
5
  import re
6
+ from typing import Any, List, Optional
7
 
8
  logger = logging.getLogger(__name__)
9
 
 
39
  pass
40
 
41
  return blocks
42
+
43
+
44
+ def extract_single_json(text: str) -> Optional[Any]:
45
+ """Extract a single JSON value from text, optimized for structured output.
46
+
47
+ Tries in order:
48
+ 1. Parse entire text as JSON
49
+ 2. Extract from ```json code fences
50
+ 3. Find first {..} or [..] block
51
+ """
52
+ blocks = extract_json_blocks(text)
53
+ return blocks[0] if blocks else None
app/utils/schema_utils.py ADDED
@@ -0,0 +1,156 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ from __future__ import annotations
2
+
3
+ import json
4
+ import logging
5
+ from typing import Any, Dict, Optional, Tuple
6
+
7
+ from jsonschema import Draft202012Validator, SchemaError, ValidationError
8
+
9
+ logger = logging.getLogger(__name__)
10
+
11
+
12
+ def validate_response_format(body: Dict[str, Any]) -> Optional[Dict[str, Any]]:
13
+ """Validate the response_format field from the request body.
14
+
15
+ Accepts:
16
+ - {"type": "json_object"}
17
+ - {"type": "json_schema", "json_schema": {"name": "...", "strict": true, "schema": {...}}}
18
+
19
+ Returns the validated dict or None if response_format is absent.
20
+ Raises ValueError with a human-readable message on invalid input.
21
+ """
22
+ rf = body.get("response_format")
23
+ if rf is None:
24
+ return None
25
+
26
+ if not isinstance(rf, dict):
27
+ raise ValueError("response_format must be an object")
28
+
29
+ rf_type = rf.get("type")
30
+ if rf_type not in ("json_object", "json_schema"):
31
+ raise ValueError(
32
+ f"response_format.type must be 'json_object' or 'json_schema', got '{rf_type}'"
33
+ )
34
+
35
+ if rf_type == "json_object":
36
+ return rf
37
+
38
+ # json_schema path - validate the nested structure
39
+ js = rf.get("json_schema")
40
+ if not isinstance(js, dict):
41
+ raise ValueError("response_format.json_schema must be an object")
42
+
43
+ name = js.get("name")
44
+ if not isinstance(name, str) or not name.strip():
45
+ raise ValueError("json_schema.name must be a non-empty string")
46
+
47
+ schema = js.get("schema")
48
+ if not isinstance(schema, dict):
49
+ raise ValueError("json_schema.schema must be a JSON Schema object")
50
+
51
+ strict = js.get("strict")
52
+ if strict is not None and not isinstance(strict, bool):
53
+ raise ValueError("json_schema.strict must be a boolean")
54
+
55
+ # Validate the schema itself is a valid JSON Schema
56
+ try:
57
+ Draft202012Validator.check_schema(schema)
58
+ except (ValidationError, SchemaError) as e:
59
+ raise ValueError(f"json_schema.schema is not a valid JSON Schema: {e.message}")
60
+
61
+ return rf
62
+
63
+
64
+ def generate_schema_prompt(
65
+ schema: Dict[str, Any], name: str = "response"
66
+ ) -> str:
67
+ """Generate a system prompt that instructs the LLM to produce output
68
+ conforming to the given JSON Schema."""
69
+ schema_json = json.dumps(schema, indent=2)
70
+ field_instructions = _build_field_instructions(schema)
71
+
72
+ return (
73
+ f'You MUST respond with a single valid JSON object that conforms EXACTLY '
74
+ f'to this JSON Schema named "{name}":\n\n'
75
+ f"```json\n{schema_json}\n```\n\n"
76
+ "CRITICAL RULES:\n"
77
+ "1. Your entire response must be ONLY a JSON object. No text before or after.\n"
78
+ "2. Do NOT wrap the JSON in markdown code fences or any other formatting.\n"
79
+ "3. Every required field MUST be present in your response.\n"
80
+ "4. Use ONLY the types specified in the schema (string, number, integer, boolean, array, object, null).\n"
81
+ "5. Do NOT include any fields that are not defined in the schema properties.\n"
82
+ '6. For "enum" fields, use EXACTLY one of the specified values.\n'
83
+ '7. For "const" fields, use the exact specified value.\n'
84
+ "8. For nested objects, follow the sub-schema recursively.\n\n"
85
+ f"{field_instructions}\n\n"
86
+ "Respond with ONLY the JSON object - no explanation, no markdown, no code fences."
87
+ )
88
+
89
+
90
+ def _build_field_instructions(
91
+ schema: Dict[str, Any], prefix: str = ""
92
+ ) -> str:
93
+ """Recursively build human-readable field instructions from a schema."""
94
+ lines: list[str] = []
95
+ props = schema.get("properties", {})
96
+ required = set(schema.get("required", []))
97
+
98
+ for field_name, field_schema in props.items():
99
+ full_name = f"{prefix}{field_name}" if prefix else field_name
100
+ field_type = field_schema.get("type", "any")
101
+ is_required = field_name in required
102
+
103
+ status = "REQUIRED" if is_required else "optional"
104
+ desc = field_schema.get("description", "")
105
+
106
+ if field_type == "string" and "enum" in field_schema:
107
+ enum_vals = ", ".join(f'"{v}"' for v in field_schema["enum"])
108
+ line = f'- Field "{full_name}" ({status}): Must be one of [{enum_vals}].'
109
+ elif field_type == "string" and "format" in field_schema:
110
+ fmt = field_schema["format"]
111
+ line = f'- Field "{full_name}" ({status}): type=string, format="{fmt}".'
112
+ elif field_type == "array" and "items" in field_schema:
113
+ items = field_schema["items"]
114
+ item_type = items.get("type", "any")
115
+ line = f'- Field "{full_name}" ({status}): type=array of {item_type}.'
116
+ elif field_type == "object" and "properties" in field_schema:
117
+ nested = _build_field_instructions(field_schema, prefix=f"{full_name}.")
118
+ if nested:
119
+ lines.append(f'- Field "{full_name}" ({status}): nested object with fields:')
120
+ lines.append(nested)
121
+ continue
122
+ elif field_type == "null":
123
+ line = f'- Field "{full_name}" ({status}): can be null.'
124
+ else:
125
+ line = f'- Field "{full_name}" ({status}): type={field_type}.'
126
+
127
+ if desc:
128
+ line = line.rstrip(".") + f". {desc}"
129
+ lines.append(line)
130
+
131
+ if not schema.get("additionalProperties", True):
132
+ lines.append("- Do NOT include any additional properties not listed above.")
133
+
134
+ return "\n".join(lines) if lines else ""
135
+
136
+
137
+ def validate_against_schema(
138
+ data: Any, schema: Dict[str, Any]
139
+ ) -> Tuple[bool, Any, Optional[str]]:
140
+ """Validate data against a JSON Schema.
141
+
142
+ Returns:
143
+ (is_valid, data, error_message_or_None)
144
+ """
145
+ try:
146
+ validator = Draft202012Validator(schema)
147
+ validator.validate(data)
148
+ return True, data, None
149
+ except ValidationError as e:
150
+ path = (
151
+ ".".join(str(p) for p in e.absolute_path)
152
+ if e.absolute_path
153
+ else "(root)"
154
+ )
155
+ msg = f"Validation error at '{path}': {e.message}"
156
+ return False, data, msg
pyproject.toml CHANGED
@@ -31,6 +31,7 @@ dependencies = [
31
  "seaborn>=0.13.0",
32
  "spacy>=3.7.0",
33
  "phonenumbers>=8.13.0",
 
34
  ]
35
 
36
  [project.optional-dependencies]
 
31
  "seaborn>=0.13.0",
32
  "spacy>=3.7.0",
33
  "phonenumbers>=8.13.0",
34
+ "jsonschema>=4.21.0",
35
  ]
36
 
37
  [project.optional-dependencies]
requirements.txt CHANGED
@@ -55,3 +55,6 @@ chardet>=5.2.0
55
  clean-text>=0.6.0
56
  text-unidecode>=1.3
57
  Unidecode>=1.3.8
 
 
 
 
55
  clean-text>=0.6.0
56
  text-unidecode>=1.3
57
  Unidecode>=1.3.8
58
+
59
+ # JSON Schema validation
60
+ jsonschema>=4.21.0