from __future__ import annotations import json import re import time from typing import Any, Dict, List, Optional, Tuple from app.core.logger import get_logger logger = get_logger(__name__) MAX_CONTENT_LENGTH = 10_000_000 MAX_REPAIR_PASSES = 6 MAX_RESULTS = 100 class ExtractionResult: def __init__( self, success: bool, data: List[Any], time_ms: float, error_message: Optional[str] = None, extraction_method: Optional[str] = None, total_extracted: int = 0, input_length: int = 0, ): self.success = success self.data = data self.time_ms = time_ms self.error_message = error_message self.extraction_method = extraction_method self.total_extracted = total_extracted self.input_length = input_length def to_dict(self) -> Dict[str, Any]: return { "success": self.success, "data": self.data, "time_ms": self.time_ms, "error_message": self.error_message, "extraction_method": self.extraction_method, "total_extracted": self.total_extracted, "input_length": self.input_length, } def _purge_nan_inf(obj: Any) -> Any: if isinstance(obj, float): if obj != obj or obj == float("inf") or obj == -float("inf"): return None return obj if isinstance(obj, dict): return {k: _purge_nan_inf(v) for k, v in obj.items()} if isinstance(obj, list): return [_purge_nan_inf(v) for v in obj] return obj def _safe_json_parse(raw: str) -> Optional[Any]: if not raw or len(raw) == 0: return None try: parsed = json.loads(raw) return _purge_nan_inf(parsed) except (json.JSONDecodeError, ValueError): return None def _is_meaningful(value: Any) -> bool: if value is None: return False if isinstance(value, bool): return True if isinstance(value, (int, float)): return True if isinstance(value, str): return len(value.strip()) > 0 if isinstance(value, (list, tuple)): return len(value) > 0 if isinstance(value, dict): return len(value) > 0 return False def _is_trivial(value: Any) -> bool: if isinstance(value, (list, tuple)): return len(value) == 0 if isinstance(value, dict): return len(value) == 0 return False def _normalize_whitespace(raw: str) -> str: result = raw result = result.replace("\r\n", "\n") result = result.replace("\r", "\n") result = result.replace("\t", " ") result = result.replace("\u00a0", " ") result = re.sub(r"[\u200b-\u200d]", "", result) result = result.replace("\ufeff", "") return result def _strip_bom(raw: str) -> str: if raw and ord(raw[0]) == 0xFEFF: return raw[1:] return raw def _repair_trailing_commas(raw: str) -> str: result = raw prev = None passes = 0 while result != prev and passes < MAX_REPAIR_PASSES: prev = result result = re.sub(r",(\s*[}\]])", r"\1", result) passes += 1 return result def _repair_leading_commas(raw: str) -> str: return re.sub(r"([\[{])\s*,", r"\1", raw) def _repair_double_commas(raw: str) -> str: return re.sub(r",(\s*),", r",\1", raw) def _quote_unquoted_keys(raw: str) -> str: return re.sub(r'([{,]\s*)([A-Za-z_$][A-Za-z0-9_$]*)\s*:', r'\1"\2":', raw) def _replace_single_quote_strings(raw: str) -> str: def _replace(m: re.Match) -> str: inner = m.group(1) escaped = inner.replace('"', '\\"') return f': "{escaped}"' return re.sub(r":\s*'((?:[^'\\]|\\.)*)'", _replace, raw) def _replace_single_quote_keys(raw: str) -> str: def _replace(m: re.Match) -> str: pre, key, post = m.group(1), m.group(2), m.group(3) escaped = key.replace('"', '\\"') return f'{pre}"{escaped}"{post}' return re.sub(r"([{,]\s*)'((?:[^'\\]|\\.)*)'(\s*:)", _replace, raw) def _fix_ellipsis_values(raw: str) -> str: return re.sub(r":\s*\.\.\.", ": null", raw) def _fix_undefined_values(raw: str) -> str: return re.sub(r":\s*undefined\b", ": null", raw, flags=re.IGNORECASE) def _fix_nan_values(raw: str) -> str: return re.sub(r":\s*NaN\b", ": null", raw) def _fix_infinity_values(raw: str) -> str: return re.sub(r":\s*-?Infinity\b", ": null", raw) def _fix_hex_numbers(raw: str) -> str: def _replace(m: re.Match) -> str: hex_val = m.group(1) return f": {int(hex_val, 16)}" return re.sub(r":\s*(0x[0-9a-fA-F]+)", _replace, raw) def _strip_js_comments(raw: str) -> str: result = re.sub(r"//[^\n]*", "", raw) result = re.sub(r"/\*[\s\S]*?\*/", "", result) return result def _remove_bare_string_entries(raw: str) -> str: lines = raw.split("\n") cleaned: List[str] = [] for i, line in enumerate(lines): trimmed = line.strip() is_bare = bool(re.match(r'^"[^"]*",?\s*$', trimmed)) and ":" not in trimmed if is_bare: if cleaned: cleaned[-1] = re.sub(r",\s*$", "", cleaned[-1]) continue cleaned.append(line) return "\n".join(cleaned) def _fix_single_element_bare_objects(raw: str) -> str: def _replace(m: re.Match) -> str: content = m.group(1) if ":" in content: return m.group(0) return "{}" return re.sub(r'\{\s*"([^"]+)"\s*\}', _replace, raw) def _fix_missing_commas(raw: str) -> str: result = raw result = re.sub(r'("\s*)\n(\s*")', r'\1,\n\2', result) result = re.sub(r"(\d)\n(\s*\")", r'\1,\n\2', result) result = re.sub(r'("\s*)\n(\s*\d)', r'\1,\n\2', result) result = re.sub(r"(\})\n(\s*\{)", r'\1,\n\2', result) result = re.sub(r"(\])\n(\s*\[)", r'\1,\n\2', result) return result def _apply_repair_pipeline(raw: str) -> str: result = raw result = _strip_js_comments(result) result = _remove_bare_string_entries(result) result = _fix_single_element_bare_objects(result) result = _replace_single_quote_keys(result) result = _replace_single_quote_strings(result) result = _quote_unquoted_keys(result) result = _fix_ellipsis_values(result) result = _fix_undefined_values(result) result = _fix_nan_values(result) result = _fix_infinity_values(result) result = _fix_hex_numbers(result) result = _repair_leading_commas(result) result = _repair_trailing_commas(result) result = _repair_double_commas(result) result = _fix_missing_commas(result) return result def _truncate_to_balanced(raw: str) -> str: if not raw: return raw opener = raw[0] if opener not in ("{", "["): return raw closer = "}" if opener == "{" else "]" depth = 0 in_string = False escape = False for i, char in enumerate(raw): if in_string: if escape: escape = False elif char == "\\": escape = True elif char == '"': in_string = False continue if char == '"': in_string = True continue if char == opener: depth += 1 elif char == closer: depth -= 1 if depth == 0: return raw[: i + 1] return raw def _close_unclosed_structures(raw: str) -> str: stack: List[str] = [] in_string = False escape = False for char in raw: if in_string: if escape: escape = False elif char == "\\": escape = True elif char == '"': in_string = False continue if char == '"': in_string = True elif char == "{": stack.append("}") elif char == "[": stack.append("]") elif char == "}" or char == "]": if stack and stack[-1] == char: stack.pop() if not stack: return raw result = raw.rstrip() result = re.sub(r",\s*$", "", result) for closer in reversed(stack): result += closer return result def _try_parse_with_repair(raw: str) -> Optional[Any]: trimmed = raw.strip() if not trimmed: return None direct = _safe_json_parse(trimmed) if direct is not None: return direct repaired = _apply_repair_pipeline(trimmed) after_repair = _safe_json_parse(repaired) if after_repair is not None: return after_repair truncated = _truncate_to_balanced(repaired) after_truncate = _safe_json_parse(truncated) if after_truncate is not None: return after_truncate closed = _close_unclosed_structures(repaired) after_close = _safe_json_parse(closed) if after_close is not None: return after_close closed_truncated = _close_unclosed_structures(truncated) return _safe_json_parse(closed_truncated) def _find_balanced_closing(text: str, start: int) -> int: opener = text[start] if opener not in ("{", "["): return -1 closer = "}" if opener == "{" else "]" depth = 0 in_string = False escape = False for i in range(start, len(text)): char = text[i] if in_string: if escape: escape = False elif char == "\\": escape = True elif char == '"': in_string = False continue if char == '"': in_string = True continue if char == opener: depth += 1 elif char == closer: depth -= 1 if depth == 0: return i return -1 def _extract_from_fenced_blocks(content: str) -> Tuple[List[Any], List[Tuple[int, int]]]: results: List[Any] = [] covered_ranges: List[Tuple[int, int]] = [] patterns = [ re.compile(r"```json\s*\n?(.*?)```", re.DOTALL), re.compile(r"```javascript\s*\n?(.*?)```", re.DOTALL), re.compile(r"```js\s*\n?(.*?)```", re.DOTALL), re.compile(r"```typescript\s*\n?(.*?)```", re.DOTALL), re.compile(r"```ts\s*\n?(.*?)```", re.DOTALL), re.compile(r"```(.*?)```", re.DOTALL), re.compile(r"~~~json\s*\n?(.*?)~~~", re.DOTALL), re.compile(r"~~~(.*?)~~~", re.DOTALL), ] seen_ranges: set = set() for pattern in patterns: for match in pattern.finditer(content): range_key = (match.start(), match.end()) if range_key in seen_ranges: continue seen_ranges.add(range_key) raw = match.group(1).strip() if match.lastindex else match.group(1).strip() if not raw: continue parsed = _try_parse_with_repair(raw) if parsed is not None and not _is_trivial(parsed): results.append(parsed) covered_ranges.append(range_key) return results, covered_ranges def _extract_from_json_tags(content: str) -> Tuple[List[Any], List[Tuple[int, int]]]: results: List[Any] = [] covered_ranges: List[Tuple[int, int]] = [] pattern = re.compile(r"]*>(.*?)", re.DOTALL) for match in pattern.finditer(content): raw = match.group(1).strip() if not raw: continue parsed = _try_parse_with_repair(raw) if parsed is not None and not _is_trivial(parsed): results.append(parsed) covered_ranges.append((match.start(), match.end())) return results, covered_ranges def _is_inside_range(index: int, ranges: List[Tuple[int, int]]) -> bool: for start, end in ranges: if start <= index <= end: return True return False def _extract_balanced_json(content: str, skip_ranges: List[Tuple[int, int]]) -> List[Any]: results: List[Any] = [] cursor = 0 while cursor < len(content): if _is_inside_range(cursor, skip_ranges): cursor += 1 continue char = content[cursor] if char not in ("{", "["): cursor += 1 continue end = _find_balanced_closing(content, cursor) if end != -1: candidate = content[cursor : end + 1] if len(candidate) >= 2: parsed = _try_parse_with_repair(candidate) if parsed is not None and not _is_trivial(parsed): results.append(parsed) cursor = end + 1 continue else: partial = content[cursor:] if len(partial) >= 2: parsed = _try_parse_with_repair(partial) if parsed is not None and not _is_trivial(parsed): results.append(parsed) break cursor += 1 return results def _extract_json_lines(content: str, skip_ranges: List[Tuple[int, int]]) -> List[Any]: results: List[Any] = [] offset = 0 for line in content.split("\n"): line_start = offset offset += len(line) + 1 if _is_inside_range(line_start, skip_ranges): continue trimmed = line.strip() if not trimmed.startswith("{") and not trimmed.startswith("["): continue parsed = _safe_json_parse(trimmed) if parsed is not None and not _is_trivial(parsed): results.append(parsed) return results def _extract_entire_content(content: str) -> List[Any]: trimmed = content.strip() if not trimmed.startswith("{") and not trimmed.startswith("["): return [] parsed = _try_parse_with_repair(trimmed) if parsed is not None and not _is_trivial(parsed): return [parsed] return [] def _deduplicate(items: List[Any]) -> List[Any]: seen: set = set() result: List[Any] = [] for item in items: key = json.dumps(item, sort_keys=True, default=str) if key not in seen: seen.add(key) result.append(item) return result def _remove_contained_subsets(items: List[Any]) -> List[Any]: serialized = [json.dumps(item, sort_keys=True, default=str) for item in items] result: List[Any] = [] for i, current in enumerate(serialized): if not current: continue is_contained = any( j != i and other and len(other) > len(current) and current in other for j, other in enumerate(serialized) ) if not is_contained: result.append(items[i]) return result def extract_json_from_content(content: Any, limit: Optional[int] = None) -> List[Any]: if not isinstance(content, str): return [] normalized = _strip_bom(_normalize_whitespace(content)) if len(normalized) == 0: return [] if len(normalized) > MAX_CONTENT_LENGTH: logger.warning("Content exceeds maximum length of %d", MAX_CONTENT_LENGTH) return [] entire = _extract_entire_content(normalized) if entire: filtered = [v for v in entire if _is_meaningful(v)] return filtered[:limit] if limit else filtered fenced_results, fenced_ranges = _extract_from_fenced_blocks(normalized) tag_results, tag_ranges = _extract_from_json_tags(normalized) all_skip_ranges = fenced_ranges + tag_ranges balanced_results = _extract_balanced_json(normalized, all_skip_ranges) line_results = _extract_json_lines(normalized, all_skip_ranges) combined = fenced_results + tag_results + balanced_results + line_results meaningful = [v for v in combined if _is_meaningful(v)] deduplicated = _deduplicate(meaningful) filtered = _remove_contained_subsets(deduplicated) return filtered[:limit] if limit else filtered def extract_first_json(content: Any) -> Optional[Any]: results = extract_json_from_content(content, limit=1) return results[0] if results else None def extract_json(content: Any, limit: Optional[int] = None) -> ExtractionResult: start = time.perf_counter() try: data = extract_json_from_content(content, limit) elapsed = round((time.perf_counter() - start) * 1000, 3) method: Optional[str] = None if data: if isinstance(content, str): trimmed = content.strip() if trimmed.startswith("{") or trimmed.startswith("["): method = "entire-content" elif "```" in content or "~~~" in content: method = "fenced-blocks" elif " 0, data=data, time_ms=elapsed, error_message=None if data else "No JSON content could be extracted", extraction_method=method, total_extracted=len(data), input_length=len(content) if isinstance(content, str) else 0, ) except Exception as exc: elapsed = round((time.perf_counter() - start) * 1000, 3) logger.exception("extract_json failed") return ExtractionResult( success=False, data=[], time_ms=elapsed, error_message=str(exc), extraction_method=None, total_extracted=0, input_length=len(content) if isinstance(content, str) else 0, )