src/hub.py

import io
import json

import httpx
from huggingface_hub import HfFileSystem, hf_hub_url
from huggingface_hub.utils import build_hf_headers


client = httpx.AsyncClient(follow_redirects=True)
fs = HfFileSystem()


def glob(path):
    paths = fs.glob(path)
    return paths


async def load_json_file(path):
    url = to_url(path)
    r = await client.get(url)
    return r.json()


async def load_jsonlines_file(path):
    url = to_url(path)
    r = await client.get(url, headers=build_hf_headers())
    f = io.StringIO(r.text)
    return [json.loads(line) for line in f]


def to_url(path):
    _, org_name, ds_name, filename = path.split("/", 3)
    return hf_hub_url(repo_id=f"{org_name}/{ds_name}", filename=filename, repo_type="dataset")