File size: 7,910 Bytes
52585d7
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
"""์‹ค์ œ P ์—ฐ์† ์ˆ˜์‹์˜ ๊ถŒ๋ฆฌยท๋ถ„ํ• ยทsymbol-group ๊ณ„์•ฝ์„ fail-closed๋กœ ๊ฒ€์‚ฌํ•œ๋‹ค."""

from __future__ import annotations

from collections import defaultdict
from collections.abc import Iterable, Sequence
from typing import Any


REQUIRED_SPLITS_06 = frozenset({"training", "validation", "test"})
REQUIRED_RECORD_FIELDS_06 = (
    "formula_id", "origin_id", "writer_id", "device_id", "source_id", "split",
    "canvas_width", "canvas_height", "symbols", "rights_track",
    "commercial_training_allowed",
)


def _identity_overlap06(records: Sequence[dict[str, Any]], field: str) -> dict[str, list[str]]:
    """ํ•„์š” ๋ณ€์ˆ˜: formula recordยทidentity field. ์ž‘๋™ ์›๋ฆฌ: ํ•˜๋‚˜์˜ identity๊ฐ€ ๋‘˜ ์ด์ƒ์˜ split์— ๋‚˜ํƒ€๋‚œ ๊ฒฝ์šฐ๋งŒ ๋ฐ˜ํ™˜ํ•œ๋‹ค."""

    split_by_identity: dict[str, set[str]] = defaultdict(set)
    for record in records:
        value = str(record.get(field) or "").strip()
        if value:
            split_by_identity[value].add(str(record.get("split") or ""))
    return {
        identity: sorted(splits)
        for identity, splits in split_by_identity.items()
        if len(splits) > 1
    }


def _valid_strokes06(strokes: Any) -> bool:
    """ํ•„์š” ๋ณ€์ˆ˜: symbol stroke payload. ์ž‘๋™ ์›๋ฆฌ: ๋น„์–ด ์žˆ์ง€ ์•Š์€ ํš๊ณผ ์œ ํ•œ ์ขŒํ‘œ ํ•„๋“œ๋ฅผ ๊ตฌ์กฐ์ ์œผ๋กœ ๊ฒ€์‚ฌํ•œ๋‹ค."""

    if not isinstance(strokes, list) or not strokes:
        return False
    for stroke in strokes:
        if not isinstance(stroke, list) or not stroke:
            return False
        for point in stroke:
            if not isinstance(point, dict) or "x" not in point or "y" not in point:
                return False
            try:
                x, y = float(point["x"]), float(point["y"])
            except (TypeError, ValueError):
                return False
            if not (-1e9 < x < 1e9 and -1e9 < y < 1e9):
                return False
    return True


def audit_p_formula_records06(
    records: Sequence[dict[str, Any]], *, minimum_independent_sources: int = 2,
) -> dict[str, Any]:
    """ํ•„์š” ๋ณ€์ˆ˜: ์‹ค์ œ ์—ฐ์†์‹ record. ์ž‘๋™ ์›๋ฆฌ: P ๊ถŒ๋ฆฌยทํ•„์ˆ˜ metadataยทidentity ๋ˆ„์ˆ˜ยทgroup ์ •๋‹ต์„ ํ•œ ๋ฒˆ์— ๊ฐ์‚ฌํ•œ๋‹ค."""

    issues: list[dict[str, Any]] = []
    formula_ids: set[str] = set()
    origin_splits: dict[str, set[str]] = defaultdict(set)
    split_counts: dict[str, int] = defaultdict(int)
    symbol_counts: dict[str, int] = defaultdict(int)
    missing_timestamp_symbols = missing_pressure_symbols = total_symbols = 0

    for index, record in enumerate(records):
        missing = [
            field for field in REQUIRED_RECORD_FIELDS_06
            if field not in record or record[field] is None or record[field] == ""
        ]
        if missing:
            issues.append({"type": "missing_fields", "record_index": index, "fields": missing})
            continue
        formula_id = str(record["formula_id"])
        if formula_id in formula_ids:
            issues.append({"type": "duplicate_formula_id", "formula_id": formula_id})
        formula_ids.add(formula_id)
        split = str(record["split"])
        split_counts[split] += 1
        if split not in REQUIRED_SPLITS_06:
            issues.append({"type": "invalid_split", "formula_id": formula_id, "split": split})
        origin_splits[str(record["origin_id"])].add(split)
        if str(record["rights_track"]) != "P" or record["commercial_training_allowed"] is not True:
            issues.append({"type": "rights_not_product_approved", "formula_id": formula_id})
        try:
            if float(record["canvas_width"]) <= 0 or float(record["canvas_height"]) <= 0:
                raise ValueError
        except (TypeError, ValueError):
            issues.append({"type": "invalid_canvas", "formula_id": formula_id})

        symbols = record["symbols"]
        if not isinstance(symbols, list) or not symbols:
            issues.append({"type": "empty_symbol_groups", "formula_id": formula_id})
            continue
        for symbol_index, symbol in enumerate(symbols):
            total_symbols += 1
            token = str(symbol.get("token") or "").strip() if isinstance(symbol, dict) else ""
            if not token or not isinstance(symbol, dict) or not _valid_strokes06(symbol.get("strokes")):
                issues.append({
                    "type": "invalid_symbol_group", "formula_id": formula_id,
                    "symbol_index": symbol_index,
                })
                continue
            symbol_counts[token] += 1
            points = [point for stroke in symbol["strokes"] for point in stroke]
            if not all(point.get("t") is not None for point in points):
                missing_timestamp_symbols += 1
            if not all(point.get("pressure") is not None for point in points):
                missing_pressure_symbols += 1

    for origin, splits in origin_splits.items():
        if len(splits) > 1:
            issues.append({"type": "origin_split_leakage", "origin_id": origin, "splits": sorted(splits)})
    identity_overlap = {
        field: _identity_overlap06(records, field)
        for field in ("writer_id", "device_id", "source_id")
    }
    for field, overlaps in identity_overlap.items():
        if overlaps:
            issues.append({
                "type": f"{field}_split_leakage",
                "count": len(overlaps),
                "examples": dict(list(overlaps.items())[:10]),
            })
    missing_splits = sorted(REQUIRED_SPLITS_06 - set(split_counts))
    if missing_splits:
        issues.append({"type": "missing_required_splits", "splits": missing_splits})
    independent_sources = len({str(record.get("source_id") or "") for record in records if record.get("source_id")})
    if independent_sources < minimum_independent_sources:
        issues.append({
            "type": "insufficient_independent_sources",
            "observed": independent_sources, "required": minimum_independent_sources,
        })

    return {
        "schema": "aiflow-p-formula-preflight-v1",
        "records": len(records),
        "symbols": total_symbols,
        "split_counts": dict(sorted(split_counts.items())),
        "label_count": len(symbol_counts),
        "independent_sources": independent_sources,
        "identity_overlap_counts": {
            field: len(overlaps) for field, overlaps in identity_overlap.items()
        },
        "missing_metadata_slices": {
            "timestamp_symbols": missing_timestamp_symbols,
            "pressure_symbols": missing_pressure_symbols,
        },
        "issues": issues,
        "eligible_for_product_evaluation": not issues,
        "product_validation": False,
    }


def formula_boundary_candidates06(record: dict[str, Any]) -> list[dict[str, Any]]:
    """ํ•„์š” ๋ณ€์ˆ˜: ๊ฒ€์ฆ๋œ formula record. ์ž‘๋™ ์›๋ฆฌ: ์‹ค์ œ ๋‹จ์ผ group๊ณผ ์ธ์ ‘ ๋‘ group ๊ฒฐํ•ฉ์„ boundary ์ •๋‹ต ํ›„๋ณด๋กœ ๋งŒ๋“ ๋‹ค."""

    symbols = record["symbols"]
    candidates = [
        {
            "formula_id": str(record["formula_id"]),
            "symbol_indices": [index],
            "strokes": symbol["strokes"],
            "boundary_target": 0,
        }
        for index, symbol in enumerate(symbols)
    ]
    for index in range(len(symbols) - 1):
        candidates.append({
            "formula_id": str(record["formula_id"]),
            "symbol_indices": [index, index + 1],
            "strokes": [*symbols[index]["strokes"], *symbols[index + 1]["strokes"]],
            "boundary_target": 1,
        })
    return candidates


def collect_formula_boundary_candidates06(
    records: Iterable[dict[str, Any]],
) -> list[dict[str, Any]]:
    """ํ•„์š” ๋ณ€์ˆ˜: ๊ฒ€์ฆ๋œ formula iterable. ์ž‘๋™ ์›๋ฆฌ: formula๋ณ„ ์‹ค์ œ group ํ›„๋ณด๋ฅผ ํ‰๊ฐ€์šฉ ํ‰ํƒ„ ๋ชฉ๋ก์œผ๋กœ ๋งŒ๋“ ๋‹ค."""

    return [
        candidate
        for record in records
        for candidate in formula_boundary_candidates06(record)
    ]