Create dataset_manager.py
Browse files- dataset_manager.py +24 -0
dataset_manager.py
ADDED
|
@@ -0,0 +1,24 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
from datasets import load_dataset
|
| 2 |
+
from huggingface_hub import HfApi, login
|
| 3 |
+
|
| 4 |
+
# Hugging Face token'ınızı ayarlayın (Settings > Access Tokens)
|
| 5 |
+
HF_TOKEN = "your_hf_token_here"
|
| 6 |
+
DATASET_REPO = "anime-backend/data"
|
| 7 |
+
|
| 8 |
+
def get_dataset():
|
| 9 |
+
try:
|
| 10 |
+
# Dataset'i yükle (eğer yoksa oluştur)
|
| 11 |
+
return load_dataset(DATASET_REPO, split='train')
|
| 12 |
+
except:
|
| 13 |
+
# Yeni dataset oluştur
|
| 14 |
+
api = HfApi(token=HF_TOKEN)
|
| 15 |
+
api.create_repo(repo_id=DATASET_REPO, repo_type="dataset", private=True)
|
| 16 |
+
|
| 17 |
+
# Boş dataset ile başlat
|
| 18 |
+
dummy_data = {"users": []}
|
| 19 |
+
return load_dataset(DATASET_REPO, data_files=dummy_data, split='train')
|
| 20 |
+
|
| 21 |
+
def save_to_dataset(data):
|
| 22 |
+
dataset = get_dataset()
|
| 23 |
+
updated_data = dataset.add_item(data) # Yeni veri ekle
|
| 24 |
+
updated_data.push_to_hub(DATASET_REPO)
|