| import random |
| import os |
| import sys |
| import json |
| import csv |
| from huggingface_hub import hf_hub_download, HfApi |
|
|
| |
| sys.stdout.reconfigure(encoding='utf-8') |
|
|
| |
| sys.path.append(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) |
|
|
| from config import AppConfig |
|
|
| def main(): |
| repo_id = "toecm/PureChain_Dataset" |
| filename = "pending_tx_queue.json" |
| queue = [] |
| |
| from huggingface_hub import get_token |
| hf_token = getattr(AppConfig, 'HF_TOKEN', None) or get_token() |
| if not hf_token: |
| print("❌ Missing HF_TOKEN in config and Hugging Face CLI.") |
| return |
|
|
| print("☁️ Downloading existing pending_tx_queue.json from Hugging Face...") |
| try: |
| downloaded_path = hf_hub_download( |
| repo_id=repo_id, |
| repo_type="dataset", |
| filename=filename, |
| token=hf_token |
| ) |
| with open(downloaded_path, "r", encoding='utf-8') as f: |
| queue = json.load(f) |
| print(f"✅ Downloaded existing queue with {len(queue)} items.") |
| except Exception as e: |
| print(f"ℹ️ Could not download existing queue (it may not exist). Starting fresh. ({e})") |
| queue = [] |
|
|
| korean_dialects = [ |
| "Gyeongsangdo Satoori", |
| "Jeju Satoori", |
| "Gangwon Satoori", |
| "Chungcheong Satoori", |
| "Seoul Satoori", |
| "Jeolla Satoori", |
| ] |
|
|
| print("🎲 Generating 60 Korean↔Korean interactions...") |
| for i in range(60): |
| d1 = random.choice(korean_dialects) |
| d2 = random.choice(korean_dialects) |
| payload = { |
| "original": f"Simulated utterance from {d1} to {d2}", |
| "dialect": d2, |
| "Data_Origin": f"Simulation: {d1}", |
| "user": "System Auto-Simulation" |
| } |
| queue.append(payload) |
|
|
| print("🎲 Generating 30 interactions for each Korean Satoori with American English...") |
| for dialect in korean_dialects: |
| for i in range(30): |
| payload = { |
| "original": f"Simulated utterance from {dialect} to American English", |
| "dialect": "American English", |
| "Data_Origin": f"Simulation: {dialect}", |
| "user": "System Auto-Simulation" |
| } |
| queue.append(payload) |
|
|
| print(f"✅ Generated simulations. Total queue size: {len(queue)}") |
| |
| local_path = "pending_tx_queue.json" |
| with open(local_path, "w", encoding="utf-8") as f: |
| json.dump(queue, f, indent=2, ensure_ascii=False) |
| |
| |
| csv_path = "pending_approvals.csv" |
| with open(csv_path, "w", newline="", encoding="utf-8") as csvfile: |
| fieldnames = ["original", "dialect", "Data_Origin", "user"] |
| writer = csv.DictWriter(csvfile, fieldnames=fieldnames) |
| writer.writeheader() |
| for item in queue: |
| writer.writerow({k: item.get(k, "") for k in fieldnames}) |
| |
| print("☁️ Uploading updated queue back to Hugging Face...") |
| api = HfApi(token=hf_token) |
| |
| api.upload_file( |
| path_or_fileobj=local_path, |
| path_in_repo=filename, |
| repo_id=repo_id, |
| repo_type="dataset", |
| commit_message="🤖 Auto-queued 240 simulated interactions (JSON)" |
| ) |
| |
| api.upload_file( |
| path_or_fileobj=csv_path, |
| path_in_repo="pending_approvals.csv", |
| repo_id=repo_id, |
| repo_type="dataset", |
| commit_message="🤖 Auto-queued 240 simulated interactions (CSV)" |
| ) |
| print("✅ Successfully uploaded JSON and CSV! The UI should see the new pending items shortly.") |
|
|
| if __name__ == "__main__": |
| main() |
|
|