HCAI-Lab/w2-consensus-deepdive-unlearning-artifacts / social-data-attribution-w2 /src /dolma /pool_sample /stratified.py
| """Deprecated compatibility wrapper for manifest-backed sampling.""" | |
| from __future__ import annotations | |
| import warnings | |
| from collections.abc import Sequence | |
| from dolma.constants import ( | |
| BIN_EMPTY, | |
| BIN_FULL, | |
| BIN_PARTIAL, | |
| BIN_SPARSE, | |
| FORMATS, | |
| TARGET_DOCS_PER_BIN, | |
| TOKEN_FLOOR_PER_BIN, | |
| TOPICS, | |
| ) | |
| from dolma.pool_sample.manifest_cli import main as manifest_sample_main | |
| from dolma.pool_sample.manifest_cli import parse_args | |
| from dolma.pool_sample.sampling import ( | |
| REPRESENTATIVE_TOKEN_TARGET, | |
| SEED, | |
| compute_bin_stats, | |
| generate_dummy_manifest, | |
| representative_sample, | |
| stratified_sample, | |
| ) | |
| DEPRECATION_MESSAGE = ( | |
| "dolma.pool_sample.stratified is deprecated. " | |
| "Use dolma.pool_sample.manifest_cli or the data-attribution-manifest-sample CLI." | |
| ) | |
| def main(argv: Sequence[str] | None = None) -> int: | |
| warnings.warn(DEPRECATION_MESSAGE, FutureWarning, stacklevel=2) | |
| return manifest_sample_main(argv) | |
| __all__ = [ | |
| "BIN_EMPTY", | |
| "BIN_FULL", | |
| "BIN_PARTIAL", | |
| "BIN_SPARSE", | |
| "DEPRECATION_MESSAGE", | |
| "FORMATS", | |
| "REPRESENTATIVE_TOKEN_TARGET", | |
| "SEED", | |
| "TARGET_DOCS_PER_BIN", | |
| "TOKEN_FLOOR_PER_BIN", | |
| "TOPICS", | |
| "compute_bin_stats", | |
| "generate_dummy_manifest", | |
| "main", | |
| "parse_args", | |
| "representative_sample", | |
| "stratified_sample", | |
| ] | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |
Xet Storage Details
- Size:
- 1.41 kB
- Xet hash:
- 48d55419d68f811858d88e876f7050a9ac71a41165e663feb46b13252a88bf20
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.