Update app.py

app.py

@@ -2,13 +2,33 @@ import gradio as gr
 import requests
 import bs4
 
+######## Load Database ########
+
+from huggingface_hub import HfApi, upload_file
+import json
+import uuid
+token=os.environ.get("HF_TOKEN")
+username="omnibus"
+dataset_name="tmp"
+save_data=f'https://huggingface.co/datasets/{username}/{dataset_name}/raw/main/'
+api=HfApi(token="")
+filename="test"
+
+r = requests.get(f'{save_data}crawl/{file_n}.json') 
+print(f'status code main:: {r.status_code}')
+if r.status_code==200:
+    lod = json.loads(r.text)
+    #print(f'lod:: {lod}')
+    #lod[0]['comment']=lod[0]['comment']+1
+    #lod[0]['comment_list'].append({'user':persona[persona2]['name'],'datetime':'','comment':output,'reply_list':[]})
+else:
+    lod={}
+
+#############################
+
 def sort_doc(in_list,steps_in=0,control=None):
     control_json={'control':'0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ','char':'','leng':62}
     text=str(in_list)
-
-########################################    
-    sen_list=in_list
-######################################    
     key_cnt=len(in_list)
     print(key_cnt)
     control_char=list(control_json['control'])
@@ -76,9 +96,6 @@ def sort_doc(in_list,steps_in=0,control=None):
                     print(j)
                     out_js = out_js+control_char[j]
                 sen_obj=in_list[i]    
-                #sen_obj=proc_sen(sen_list,i)
-                
-                #json_out[out_js]={'nouns':ea}
                 json_out[out_js]=sen_obj
                 print ("#################")
                 print (out_js)
@@ -186,7 +203,25 @@ def sitemap(url,level):
                 except Exception as e:
                     print (e)
         uri_key=sort_doc(link_box,8)
+######## Save Database ########
+    uid=uuid.uuid4()
+    for ea in list(uri_key.keys()):
+        if not uri_key[ea] == x for x in list(lod.values()):
+            lod[ea]=uri_key[ea]
         
+    with open(f'{uid}.json', 'w') as f:
+        json_hist=json.dumps(uri_key, indent=4)
+        f.write(json_hist)
+    f.close()
+
+    upload_file(
+        path_or_fileobj =f"{uid}.json", 
+        path_in_repo = f"crawl/{filename}.json", 
+        repo_id =f"{username}/{dataset_name}", 
+        repo_type = "dataset", 
+        token=token,
+    )
+#################################
     return link1,link2,uri_key