Buckets:
| #!/usr/bin/env python3 | |
| """Smoke-test generated STRING Parquet configs with the Hugging Face datasets API.""" | |
| import argparse | |
| import glob | |
| import sys | |
| from pathlib import Path | |
| from typing import Dict, List | |
| CONFIGS = ( | |
| "species", | |
| "protein_info", | |
| "protein_aliases", | |
| "protein_sequences", | |
| "protein_links", | |
| ) | |
| SPLITS = ("train", "validation", "test") | |
| def parse_args() -> argparse.Namespace: | |
| parser = argparse.ArgumentParser(description="Validate local or uploaded STRING HF dataset configs.") | |
| parser.add_argument( | |
| "--repo-id", | |
| default=None, | |
| help="Optional HF dataset repo id, for example LiteFold/STRING. If omitted, validates local Parquet.", | |
| ) | |
| parser.add_argument("--data-dir", type=Path, default=Path("data")) | |
| parser.add_argument("--config", choices=CONFIGS, default="protein_links") | |
| parser.add_argument("--split", default="train") | |
| parser.add_argument("--streaming", action="store_true") | |
| parser.add_argument("--preview-rows", type=int, default=3) | |
| return parser.parse_args() | |
| def local_data_files(data_dir: Path, config: str) -> Dict[str, List[str]]: | |
| files: Dict[str, List[str]] = {} | |
| for split in SPLITS: | |
| pattern = str(data_dir / config / ("%s-*.parquet" % split)) | |
| matches = sorted(glob.glob(pattern)) | |
| if matches: | |
| files[split] = matches | |
| if not files: | |
| raise SystemExit("No local Parquet files found for config %s under %s" % (config, data_dir)) | |
| return files | |
| def main() -> int: | |
| args = parse_args() | |
| try: | |
| from datasets import load_dataset | |
| except ImportError as exc: | |
| raise SystemExit( | |
| "datasets is required. Install dependencies with: python -m pip install -r requirements.txt" | |
| ) from exc | |
| if args.repo_id: | |
| dataset = load_dataset(args.repo_id, args.config, split=args.split, streaming=args.streaming) | |
| else: | |
| files = local_data_files(args.data_dir, args.config) | |
| if args.split not in files: | |
| raise SystemExit( | |
| "Split %s is not present locally for config %s. Available splits: %s" | |
| % (args.split, args.config, ", ".join(sorted(files))) | |
| ) | |
| dataset = load_dataset("parquet", data_files=files, split=args.split, streaming=args.streaming) | |
| print("config:", args.config) | |
| print("split:", args.split) | |
| print("features:", dataset.features) | |
| if args.streaming: | |
| iterator = iter(dataset) | |
| for index in range(args.preview_rows): | |
| try: | |
| print("row[%d]:" % index, next(iterator)) | |
| except StopIteration: | |
| break | |
| else: | |
| print("num_rows:", len(dataset)) | |
| for index in range(min(args.preview_rows, len(dataset))): | |
| print("row[%d]:" % index, dataset[index]) | |
| return 0 | |
| if __name__ == "__main__": | |
| sys.exit(main()) | |
Xet Storage Details
- Size:
- 2.89 kB
- Xet hash:
- 211bc314188c4aacabd538704e3dfbd8646e397e4d07d925ab819a90e29a7955
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.