File size: 2,323 Bytes
35d483e
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
"""Stable public API for data ingestion, auditing and leakage-safe splitting."""

from .audio import AudioInfo, AudioPayloadError, exact_audio_sha256, extract_audio_bytes, inspect_audio
from .audit import audit_records, build_audit_report, build_manifest_row
from .grouping import GroupingConfig, attach_group_ids, derive_group_keys, group_sizes
from .ingest import (
    DatasetReadError,
    OptionalDependencyError,
    iter_hf_records,
    iter_local_records,
    iter_parquet_records,
    iter_records,
)
from .manifest import MANIFEST_SCHEMA_VERSION, read_manifest, write_json, write_manifest
from .resolver import (
    ManifestRecordResolver,
    ManifestResolutionError,
    iter_manifest_records,
)
from .schema import (
    NormalizationResult,
    RecordValidationError,
    TurnRecord,
    ValidationIssue,
    normalize_nullable_bool,
    normalize_record,
    normalize_record_with_issues,
    validate_record,
)
from .splits import (
    DEFAULT_SPLIT_RATIOS,
    SplitValidationError,
    assert_no_split_leakage,
    assign_leave_one_out,
    assign_splits,
    build_split_report,
    find_split_leakage,
    iter_leave_one_out_folds,
    normalize_split_ratios,
    parse_split_ratios,
)

__all__ = [
    "AudioInfo",
    "AudioPayloadError",
    "DEFAULT_SPLIT_RATIOS",
    "DatasetReadError",
    "GroupingConfig",
    "MANIFEST_SCHEMA_VERSION",
    "ManifestRecordResolver",
    "ManifestResolutionError",
    "NormalizationResult",
    "OptionalDependencyError",
    "RecordValidationError",
    "SplitValidationError",
    "TurnRecord",
    "ValidationIssue",
    "assert_no_split_leakage",
    "assign_leave_one_out",
    "assign_splits",
    "attach_group_ids",
    "audit_records",
    "build_audit_report",
    "build_manifest_row",
    "build_split_report",
    "derive_group_keys",
    "exact_audio_sha256",
    "extract_audio_bytes",
    "find_split_leakage",
    "group_sizes",
    "inspect_audio",
    "iter_hf_records",
    "iter_leave_one_out_folds",
    "iter_local_records",
    "iter_manifest_records",
    "iter_parquet_records",
    "iter_records",
    "normalize_nullable_bool",
    "normalize_record",
    "normalize_record_with_issues",
    "normalize_split_ratios",
    "parse_split_ratios",
    "read_manifest",
    "validate_record",
    "write_json",
    "write_manifest",
]