deprecate clouddisk (#20)

* deprecate clouddisk and use pgsoft.pgfile instead

app.py

@@ -7,6 +7,7 @@ import gradio as gr
 from extract import extract
 import app_util
 from pgsoft.pgconst.const import service_list, functionality_list, game_list
+from pgsoft.pghost import ais
 from pgsoft.pgdate.date_utils import beijing
 import call_ai
 from str_util import normalize_text
@@ -14,7 +15,7 @@ from str_util import normalize_text
 #######################
 # proxy version
 #######################
-proxy_version = "1.0.0-2023-12-13-a"  # reconstruct ai calling
+proxy_version = "1.0.0-2024-01-02-a"  # reconstruct ai calling
 
 t = datetime.now()
 t = t.astimezone(ZoneInfo("Asia/Shanghai"))
@@ -23,28 +24,16 @@ t = t.astimezone(ZoneInfo("America/Los_Angeles"))
 print(f"[Seattle]: {t.replace(microsecond=0)}")
 
 
-ais = {
-    "b_demo_hf": "stevez/ai",
-    "pgdemo2": "stevez/ai2",
-    "pgdemo_dev": "stevez/ai_dev",
-}
-
-clouddisks = {
-    "b_demo_hf": "pgsoft/clouddisk",
-    "pgdemo2": "pgsoft/clouddisk",
-    "pgdemo_dev": "pgsoft/clouddisk_dev",
-}
-
 identity = os.environ.get("identity")
 print(f"identity: {identity}")
 if not identity:
     identity = "pgdemo_dev"
-ai = "stevez/ai_dev"
+ai = "stevez-ai-dev"
 if identity in ais:
     ai = ais[identity]
-clouddisk = "pgsoft/clouddisk_dev"
-if identity in clouddisks:
-    clouddisk = clouddisks[identity]
+db_token = os.environ.get("db_token")
+if db_token:
+    print(db_token[:5])
 
 
 def run(hf_token, service, game, functionality, nlp_command):
@@ -58,7 +47,7 @@ def run(hf_token, service, game, functionality, nlp_command):
         user_name = "__fake__"
 
     # redirect all traffic to the proxy sever
-    global ai, clouddisk
+    global ai
     if redirect is not None:
         ai = redirect
 
@@ -77,12 +66,11 @@ def run(hf_token, service, game, functionality, nlp_command):
     service_start = beijing()
     print(f"<<<<<<<<<<<<<< service starts at {service_start} <<<<<<<<<<<<<<")
     if service in ["download game", "upload game", "list games"]:
-        res = app_util.call_clouddisk(clouddisk, service, nlp_command, token)
-        res = json.loads(res)
-        if res["result"] is None:
+        res = app_util.file_service(service, nlp_command, db_token)
+        if not res:
             outp = {"status": "Failure"}
         else:
-            outp = {"status": "OK", "result": res["result"]}
+            outp = {"status": "OK", "result": res}
     else:
         assert "games" in service_list
         if service == "games":

app_util.py

@@ -2,6 +2,11 @@ import json
 
 from gradio_client import Client
 from pgsoft.pgdate.date_utils import beijing
+from pgsoft.pgfile import download, upload, list_files
+from pgsoft.pghash.md5 import md5
+from time import sleep
+from huggingface_hub import HfApi
+import os
 
 
 def call_logger(log_info, caller, hf_token) -> None:
@@ -32,19 +37,108 @@ def call_logger(log_info, caller, hf_token) -> None:
     print(f"calling logger ends at {calling_end}, costs {timecost:.2f}s")
 
 
-def call_clouddisk(url: str, service, arg, hf_token) -> str | None:
-    ##########################
-    # upload or download game
-    ##########################
-    try:
-        client = Client(
-            url,
-            hf_token=hf_token,
-            verbose=False,
+dataset_id = "hubei-hunan/games"
+local_dir = "game"
+if not os.path.exists(local_dir):
+    os.mkdir(local_dir)
+hf_api = HfApi()
+
+
+def file_service(service, arg: str, token: str):
+    """download game, upload game, or list games"""
+    if service == "download game":
+        filepath = arg.strip() + ".json"
+        res = download(
+            dataset_id,
+            filepath,
+            repo_type="dataset",
+            localdir=local_dir,
+            token=token,
+        )
+        if not res:
+            return None
+        with open(res, "r") as f:
+            outp = json.load(f)
+        print(f"[{service}] OK")
+        return outp
+    elif service == "upload game":
+        try:
+            game = json.loads(arg)
+        except json.JSONDecodeError as e:
+            print(f"[{service}] {type(e)}: {e}")
+            return None
+
+        if not isinstance(game, dict):
+            print(f"[{service}] not a dict")
+            return None
+
+        needed_keys = ["game-file", "device-id"]
+        for key in needed_keys:
+            if key not in game:
+                print(f'[{service}] error: missed "{key}"')
+                return None
+        if not isinstance(game["device-id"], str):
+            print(f'[{service}] error: "device-id" is not a str')
+            return None
+        if not isinstance(game["game-file"], dict):
+            print(f'[{service}] error: "game-file" is not a dict')
+            return None
+
+        obj = {
+            "upload-time": beijing().__str__(),
+            "game-file": game["game-file"],
+        }
+
+        maxtry = 5
+        for retry in range(maxtry):
+            md5code = md5(obj)
+            if not hf_api.file_exists(
+                repo_id=dataset_id,
+                filename=md5code + ".json",
+                repo_type="dataset",
+                token=token,
+            ):
+                break
+            sleep(0.1)
+            obj["upload-time"] = beijing().__str__()
+            maxtry -= 1
+        filename = md5code + ".json"
+        if not maxtry and hf_api.file_exists(
+            repo_id=dataset_id,
+            filename=md5code + ".json",
+            repo_type="dataset",
+            token=token,
+        ):
+            print(f"[{service}] error: file exists")
+            return None
+        localpath = os.sep.join([local_dir, filename])
+        content = json.dumps(game, indent=4)
+        with open(localpath, "w") as f:
+            f.write(content)
+        res = upload(
+            localpath,
+            filename,
+            dataset_id,
+            "dataset",
+            token,
+            f"Updated at {beijing()}",
+        )
+        if not res:
+            print(f"[{service}] error: upload failed")
+            return None
+        print(f"[{service}] OK")
+        return md5code
+    elif service == "list games":
+        games = list_files(
+            repo_id=dataset_id,
+            repo_type="dataset",
+            token=token,
         )
-        res = client.predict(service, arg, api_name="/predict")
-        print(f"[call_clouddisk] {service} from {url} [OK]")
-        return res
-    except Exception as e:
-        print(f"[call_clouddisk] {type(e)}: {e}")
+        if not games:
+            return None
+        games = {item.split(".")[0]: item for item in games if len(item) >= 32}
+        print(f"[{service}] OK")
+        return games
+    else:
+        print(f"[{service}] error: unknown service")
         return None

requirements-local.txt

@@ -57,4 +57,4 @@ tzdata==2023.3
 urllib3==2.0.6
 uvicorn==0.23.2
 websockets==11.0.3
-git+ssh://git@github.com/east-and-west-magic/pgsoft.git@tag-2023-12-01-a
+git+ssh://git@github.com/east-and-west-magic/pgsoft.git@tag-2024-01-02-a

requirements.txt

@@ -57,4 +57,4 @@ tzdata==2023.3
 urllib3==2.0.6
 uvicorn==0.23.2
 websockets==11.0.3
-git+https://github.com/east-and-west-magic/pgsoft.git@tag-2023-12-01-a
+git+https://github.com/east-and-west-magic/pgsoft.git@tag-2024-01-02-a