Spaces:
Runtime error
Runtime error
File size: 1,330 Bytes
75bc718 886047c 75bc718 886047c 75bc718 cc5c9ae 75bc718 97994ad | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 | import os
import json
import tempfile
from pathlib import Path
from huggingface_hub import CommitScheduler
from datasets import load_dataset
from util.time import TimeUtil
from util.schema import Dir
class Database:
LOCAL = os.path.join(tempfile.gettempdir(),"data")
USERNAME = "JR818"
REPO = "Web3"
def __init__(self):
try:
self.scheduler = CommitScheduler(
repo_id=self.REPO,
repo_type="dataset",
folder_path=self.LOCAL,
token=os.getenv("HF_TOKEN")
)
except Exception as e:
print(e)
def load(self,dir:Dir):
dataset = load_dataset(self.USERNAME+'/'+self.REPO,data_dir=dir.path,ignore_verifications=True,split="train")
dataset = dataset.with_format("pandas").data.to_pandas()
return dataset
def write(self,data,dir:Dir):
path = Path(self.LOCAL,dir.path)
path.mkdir(parents=True, exist_ok=True)
path = path/f'{TimeUtil.get_date()}.json'
try:
data['datetime'] = TimeUtil.get_time()
data = json.dumps(data)+"\n"
with self.scheduler.lock:
with path.open("a") as f:
f.write(data)
except Exception as e:
print(e)
database = Database()
|