uP

upload.py

@@ -0,0 +1,54 @@
+#!/usr/bin/env python3
+from huggingface_hub import HfApi
+from huggingface_hub import hf_hub_download
+import huggingface_hub
+from huggingface_hub import get_repo_discussions
+from bs4 import BeautifulSoup
+import json
+
+# repo_id = "stabilityai/stable-diffusion"
+repo_id = "huggingface-projects/diffuse-the-rest"
+dataset_repo_id = "triple-t/dummy"
+path = "/home/patrick/image_cache"
+
+print("retrieve images 0...")
+discussions_list = list(get_repo_discussions(repo_id=repo_id, repo_type="space"))
+print("retrieve images 1...")
+all_data = []
+for i, disc in enumerate(discussions_list[:5]):
+    disc = huggingface_hub.get_discussion_details(repo_id=repo_id, repo_type="space", discussion_num=disc.num)
+    page = BeautifulSoup(disc.events[0]._event["data"]["latest"]["raw"])
+    image_urls = [link.get('src') for link in page.findAll('img')]
+    data = {
+        "discussion_number": i,
+        "data": {
+          "prompt": disc.title,
+          "images": image_urls,
+        }
+    }
+    if not image_urls:
+        continue
+    else:
+        all_data.append(data)
+
+
+file_name = "_".join(repo_id.split("/")) + ".json"
+api = HfApi()
+
+path = hf_hub_download(repo_id=dataset_repo_id, filename=file_name, cache_dir=path, repo_type="dataset")
+
+with open(path, "r") as f:
+    data = json.load(path)
+
+data += all_data
+import ipdb; ipdb.set_trace()
+
+with open(path, "w") as f:
+    f.write(json.dumps(all_data, sort_keys=True, indent=4))
+
+api.upload_file(
+    path_or_fileobj=path,
+    path_in_repo=file_name,
+    repo_id=dataset_repo_id,
+    repo_type="dataset",
+)