File size: 3,264 Bytes
35d483e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | from __future__ import annotations
import sys
import unittest
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parents[1] / "src"))
from turn_detection.data import ( # noqa: E402
RecordValidationError,
normalize_nullable_bool,
normalize_record,
normalize_record_with_issues,
)
class NullableBooleanTests(unittest.TestCase):
def test_null_is_not_false(self) -> None:
self.assertIsNone(normalize_nullable_bool(None))
self.assertIsNone(normalize_nullable_bool("null"))
self.assertIs(normalize_nullable_bool(False), False)
self.assertIs(normalize_nullable_bool("0"), False)
def test_endpoint_text_aliases(self) -> None:
self.assertIs(normalize_nullable_bool("complete"), True)
self.assertIs(normalize_nullable_bool("not endpoint"), False)
def test_invalid_boolean_raises(self) -> None:
with self.assertRaises(ValueError):
normalize_nullable_bool(2, field_name="endpoint")
class RecordNormalizationTests(unittest.TestCase):
def test_upstream_schema_preserves_nullable_fillers(self) -> None:
record = normalize_record(
{
"audio": b"abc",
"id": "example-1",
"language": "en",
"endpoint_bool": True,
"midfiller": None,
"endfiller": False,
"synthetic": True,
"spoken_text": None,
"dataset": "chirp3_1",
"speaker_id": "speaker-7",
}
)
self.assertEqual(record.record_id, "example-1")
self.assertIs(record.endpoint, True)
self.assertIsNone(record.midfiller)
self.assertIs(record.endfiller, False)
self.assertEqual(record.metadata["speaker_id"], "speaker-7")
def test_invalid_required_endpoint_is_reported(self) -> None:
raw = {"audio": b"abc", "id": "bad", "endpoint_bool": "maybe"}
with self.assertRaises(RecordValidationError):
normalize_record(raw)
result = normalize_record_with_issues(raw)
self.assertIsNone(result.record.endpoint)
self.assertIn("invalid_endpoint", {issue.code for issue in result.errors})
def test_missing_id_gets_provenance_fallback(self) -> None:
result = normalize_record_with_issues(
{
"audio": b"abc",
"endpoint_bool": False,
"__source_file": "shard.parquet",
"__source_row": 9,
}
)
self.assertEqual(result.record.record_id, "shard.parquet#9")
self.assertIn("missing_id", {issue.code for issue in result.warnings})
def test_filler_flags_are_independent_and_not_coerced(self) -> None:
result = normalize_record_with_issues(
{
"audio": b"abc",
"id": "fillers",
"endpoint_bool": True,
"midfiller": True,
"endfiller": True,
}
)
self.assertTrue(result.record.midfiller)
self.assertTrue(result.record.endfiller)
self.assertNotIn("conflicting_filler_flags", {issue.code for issue in result.warnings})
if __name__ == "__main__":
unittest.main()
|