File size: 2,326 Bytes
12eff8e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 | #!/usr/bin/env python3
"""Publish Vertica↔Snowflake pair dataset to the Hugging Face Hub."""
from __future__ import annotations
import argparse
import json
from pathlib import Path
from morphsql.eval.pairs import ensure_pairs_file, load_pairs
def main() -> None:
parser = argparse.ArgumentParser(description="Publish MorphSQL pair dataset")
parser.add_argument(
"--repo",
default="dgvj-work/vertica-snowflake-pairs",
help="Hub dataset repo id (user/name)",
)
parser.add_argument("--private", action="store_true")
args = parser.parse_args()
path = ensure_pairs_file()
pairs = load_pairs()
print(f"Loaded {len(pairs)} pairs from {path}")
try:
from datasets import Dataset
from huggingface_hub import HfApi, login
except ImportError as exc:
raise SystemExit(
"Install: pip install datasets huggingface_hub\n" + str(exc)
) from exc
# Prefer token from env; login() is interactive otherwise
api = HfApi()
try:
api.whoami()
except Exception:
login()
ds = Dataset.from_list(pairs)
ds.push_to_hub(args.repo, private=args.private)
readme = f"""---
license: apache-2.0
task_categories:
- text2text-generation
language:
- en
tags:
- sql
- code
- migration
- snowflake
- vertica
- dbt
- evaluation
size_categories:
- n<1K
---
# Vertica / Oracle / Redshift / BigQuery → Snowflake SQL pairs
Synthetic + curated migration pairs for **MorphSQL** evals and fine-tuning.
- Rows: {len(pairs)}
- Fields: `id`, `category`, `source_dialect`, `target_dialect`, `source_sql`, `target_sql`, `notes`
- Categories: function, date, aggregate, ddl, ml_feature
Space: https://huggingface.co/spaces/dgvj-work/morphsql
Code: https://github.com/dgvj-work/morphsql
Author: Digvijay Waghela
"""
api.upload_file(
path_or_fileobj=readme.encode("utf-8"),
path_in_repo="README.md",
repo_id=args.repo,
repo_type="dataset",
)
# Also upload raw jsonl
api.upload_file(
path_or_fileobj=str(path),
path_in_repo="vertica_snowflake_pairs.jsonl",
repo_id=args.repo,
repo_type="dataset",
)
print(f"Published https://huggingface.co/datasets/{args.repo}")
if __name__ == "__main__":
main()
|