Buckets:
| import tempfile | |
| import unittest | |
| from pathlib import Path | |
| import pyarrow as pa | |
| import pyarrow.parquet as pq | |
| from zebra_sft.sample_raw_subset import create_subset, sample_count, selected_indices | |
| class RawSubsetTest(unittest.TestCase): | |
| def test_sampling_is_deterministic_and_config_specific(self): | |
| self.assertEqual(sample_count(49, 0.01), 1) | |
| self.assertEqual(sample_count(150, 0.01), 2) | |
| first = selected_indices(1000, 10, seed=7, config="a") | |
| self.assertEqual(first, selected_indices(1000, 10, seed=7, config="a")) | |
| self.assertNotEqual(first, selected_indices(1000, 10, seed=7, config="b")) | |
| def test_preserves_schema_and_embedded_bytes(self): | |
| with tempfile.TemporaryDirectory() as temporary: | |
| root = Path(temporary) | |
| source = root / "source" / "Config" | |
| source.mkdir(parents=True) | |
| image_type = pa.struct([("bytes", pa.binary()), ("path", pa.string())]) | |
| schema = pa.schema( | |
| [ | |
| ("Question", pa.string()), | |
| ("problem_image_1", image_type), | |
| ], | |
| metadata={b"huggingface": b"fixture"}, | |
| ) | |
| table = pa.Table.from_arrays( | |
| [ | |
| pa.array([f"q{index}" for index in range(200)]), | |
| pa.array( | |
| [ | |
| {"bytes": bytes([index % 256]), "path": f"{index}.png"} | |
| for index in range(200) | |
| ], | |
| type=image_type, | |
| ), | |
| ], | |
| schema=schema, | |
| ) | |
| pq.write_table(table.slice(0, 100), source / "train-00000-of-00002.parquet") | |
| pq.write_table(table.slice(100), source / "train-00001-of-00002.parquet") | |
| output = root / "subset" | |
| manifest = create_subset(root / "source", output, fraction=0.01, seed=7) | |
| sampled = pq.read_table(output / "Config" / "train-00000-of-00001.parquet") | |
| selected = [row["global_row"] for row in manifest["configs"][0]["selected_rows"]] | |
| self.assertEqual(sampled.schema, schema) | |
| self.assertEqual(sampled, table.take(pa.array(selected))) | |
| self.assertEqual(sampled.num_rows, 2) | |
| if __name__ == "__main__": | |
| unittest.main() | |
Xet Storage Details
- Size:
- 2.37 kB
- Xet hash:
- 94913f08a1596da68183f4e7c68156463829d6bc16ef130e9724cda7bf48bb95
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.