votes not in metadata

src/global_variables.py

@@ -1,6 +1,8 @@
 """Global variables used in the space.
 """
 
+import os
+import json
 from huggingface_hub import HfApi
 import jsonlines
 
@@ -10,23 +12,30 @@ from src.constants import DATASET_NAME, HF_TOKEN, ASSETS_FOLDER
 
 hf_api: HfApi
 all_metadata: dict
+all_votes: dict
 
 
 def setup():
     global hf_api
     global all_metadata
+    global all_votes
     hf_api = HfApi(token=HF_TOKEN)
-    hf_api.snapshot_download(
-        local_dir=f"{ASSETS_FOLDER}/{DATASET_NAME}",
-        repo_id=DATASET_NAME,
-        repo_type="dataset",
-    )
+    # hf_api.snapshot_download(
+    #     local_dir=f"{ASSETS_FOLDER}/{DATASET_NAME}",
+    #     repo_id=DATASET_NAME,
+    #     repo_type="dataset",
+    # )
     all_metadata = {}
     for split in ["train", "validation", "test"]:
         all_metadata[split] = []
         with jsonlines.open(f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/metadata.jsonl") as reader:
             for row in reader:
                 all_metadata[split].append(row)
+    all_votes = {}
+    for file in os.listdir(f"{ASSETS_FOLDER}/{DATASET_NAME}/votes"):
+        with open(f"{ASSETS_FOLDER}/{DATASET_NAME}/votes/{file}") as f:
+            key = file.split(".")[0]
+            all_votes[key] = json.load(f)
 
 def get_metadata(split):
     global all_metadata
@@ -55,5 +64,32 @@ def save_metadata(split):
         repo_type="dataset",
     )
 
+def get_votes(key):
+    global all_votes
+    global hf_api
+    try:
+        hf_api.hf_hub_download(
+            repo_id=DATASET_NAME,
+            filename=f"votes/{key}.json",
+            repo_type="dataset",
+            local_dir=f"{ASSETS_FOLDER}/{DATASET_NAME}",
+        )
+        with open(f"{ASSETS_FOLDER}/{DATASET_NAME}/votes/{key}.json") as f:
+            all_votes[key] = json.load(f)
+    except:
+        pass
+
+def save_votes(key):
+    global all_votes
+    global hf_api
+    with open(f"{ASSETS_FOLDER}/{DATASET_NAME}/votes/{key}.json", "w") as f:
+        json.dump(all_votes[key], f)
+    hf_api.upload_file(
+        path_or_fileobj=f"{ASSETS_FOLDER}/{DATASET_NAME}/votes/{key}.json",
+        path_in_repo=f"votes/{key}.json",
+        repo_id=DATASET_NAME,
+        repo_type="dataset",
+    )
+
 if gr.NO_RELOAD:
-    setup()
+    setup()

src/label_interface.py

@@ -47,7 +47,7 @@ def get_next_image(
         sample = global_variables.all_metadata[split][sample_idx]
         image_path = f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/{sample['file_name']}"
         try:
-            username_votes = sample["votes"][username]
+            username_votes = global_variables.all_votes[sample["id"]][username]
             voted_concepts = [c for c in CONCEPTS if username_votes.get(c, False)]
             unseen_concepts = [c for c in CONCEPTS if c not in username_votes]
         except KeyError:
@@ -89,19 +89,22 @@ def submit_label(
     current_split, idx = current_image.split(":")
     idx = int(idx)
     global_variables.get_metadata(current_split)
-    if "votes" not in global_variables.all_metadata[current_split][idx]:
-        global_variables.all_metadata[current_split][idx]["votes"] = {}
-    global_variables.all_metadata[current_split][idx]["votes"][username] = {c: c in voted_concepts for c in CONCEPTS}
+    s_id = global_variables.all_metadata[current_split][idx]["id"]
+    global_variables.get_votes(s_id)
+    if s_id not in global_variables.all_votes:
+        global_variables.all_votes[s_id] = {}
+    global_variables.all_votes[s_id][username] = {c: c in voted_concepts for c in CONCEPTS}
     vote_sum = {c: 0 for c in CONCEPTS}
     new_concepts = {}
     for c in CONCEPTS:
-        for vote in global_variables.all_metadata[current_split][idx]["votes"].values():
+        for vote in global_variables.all_votes[s_id].values():
             if c not in vote:
                 continue
             vote_sum[c] += 2 * vote[c] - 1
         new_concepts[c] = vote_sum[c] > 0 if vote_sum[c] != 0 else None
     global_variables.all_metadata[current_split][idx]["concepts"] = new_concepts
     global_variables.save_metadata(current_split)
+    global_variables.save_votes(s_id)
     gr.Info("Submit success")
     return get_next_image(
         split, 

src/sample_interface.py

@@ -31,7 +31,7 @@ def get_image(
     sample = global_variables.all_metadata[split][sample_idx]
     image_path = f"{ASSETS_FOLDER}/{DATASET_NAME}/data/{split}/{sample['file_name']}"
     try:
-        username_votes = sample["votes"][username]
+        username_votes = global_variables.all_votes[sample["id"]][username]
         voted_concepts = [c for c in CONCEPTS if username_votes.get(c, False)]
         unseen_concepts = [c for c in CONCEPTS if c not in username_votes]
     except KeyError:
@@ -80,19 +80,22 @@ def submit_label(
     current_split, idx = current_image.split(":")
     idx = int(idx)
     global_variables.get_metadata(current_split)
-    if "votes" not in global_variables.all_metadata[current_split][idx]:
-        global_variables.all_metadata[current_split][idx]["votes"] = {}
-    global_variables.all_metadata[current_split][idx]["votes"][username] = {c: c in voted_concepts for c in CONCEPTS}
+    s_id = global_variables.all_metadata[current_split][idx]["id"]
+    global_variables.get_votes(s_id)
+    if s_id not in global_variables.all_votes:
+        global_variables.all_votes[s_id] = {}
+    global_variables.all_votes[s_id][username] = {c: c in voted_concepts for c in CONCEPTS}
     vote_sum = {c: 0 for c in CONCEPTS}
     new_concepts = {}
     for c in CONCEPTS:
-        for vote in global_variables.all_metadata[current_split][idx]["votes"].values():
+        for vote in global_variables.all_votes[s_id].values():
             if c not in vote:
                 continue
             vote_sum[c] += 2 * vote[c] - 1
         new_concepts[c] = vote_sum[c] > 0 if vote_sum[c] != 0 else None
     global_variables.all_metadata[current_split][idx]["concepts"] = new_concepts
     global_variables.save_metadata(current_split)
+    global_variables.save_votes(s_id)
     gr.Info("Submit success")
     return get_next_image(
         split, 

src/vote_interface.py

@@ -13,8 +13,8 @@ def get_votes(
     username = profile.username
     vote_list = []
     for i,s in enumerate(global_variables.all_metadata[split]):
-        if "votes" in s and username in s["votes"]:
-            vote_list.append(f'[{i}]: {s["votes"][username]}')
+        if s["id"] in global_variables.all_votes and username in global_variables.all_votes[s["id"]]:
+            vote_list.append(f'[{i}]: {global_variables.all_votes[s["id"]][username]}')
     
     return "\n".join(vote_list)