Update app.py

app.py

@@ -302,6 +302,60 @@ def link_find(url):
     return node1,node2
 #https://huggingface.co/spaces/Omnibus/crawl
 
+def sitemap_test(url,file_state,level):
+    url_page=[]
+    url_front=[]
+    url_json=[]
+    for each_url in url:
+        uri=""
+        uri0=""
+        if url != "" and url != None:    
+            link1,link2=link_find(url)
+            if level >=2:
+                for i,ea in enumerate(link1['TREE']):
+                    print(ea)
+                    try:
+                        out_list1,out_list2=link_find(f"{uri}{ea['URL']}")
+                        link1['TREE'][i]=out_list1
+                        link2['TREE'][i]=out_list2
+                        #link1['TREE'].append(out_list)
+                        
+                        if level>=3:
+                            for n,na in enumerate(link1['TREE'][i]['TREE']):
+                                print(na)
+                                try:
+                                    out_list1,out_list2=link_find(f"{uri0}{na['URL']}")
+                                    link1['TREE'][i]['TREE'][n]=out_list1
+                                    link2['TREE'][i]['TREE'][n]=out_list2
+                                    #link1['TREE'][i]['TREE'].append(out_list1)
+                                except Exception as e:
+                                    print (e)
+                    except Exception as e:
+                        print (e)
+
+            for ea_link in link2['TREE']:
+                url_list=ea_link['URL'].split("/")
+                url_front.append(f'{url_list[1]}//{url_list[3]}')
+        uri_key=sort_doc(url_front,file_state,8)
+            
+######## Save Database ########
+    uid=uuid.uuid4()
+    with open(f'{uid}.json', 'w') as f:
+        json_hist=json.dumps(uri_key, indent=4)
+        f.write(json_hist)
+    f.close()
+
+    upload_file(
+        path_or_fileobj =f"{uid}.json", 
+        path_in_repo = f"crawl/{filename}.json", 
+        repo_id =f"{username}/{dataset_name}", 
+        repo_type = "dataset", 
+        token=token,
+    )
+#################################
+    return link1,link2,uri_key
+
+
 def sitemap(url,file_state,level):
     uri=""
     uri0=""
@@ -416,16 +470,18 @@ def sitemap_OG(url,level):
     return link1
 
 def test():
+    seed_box=[]
     with open("./seed.txt") as f:
         this = f.readlines()
     f.close()
     for ea in this:
         ea=ea.strip().strip("\n")
-        print(ea)
-        try:
-            a,b = sitemap(ea,None,1)
-        except Exception as e:
-            print (e)
+        seed_box.append(ea)
+        #print(ea)
+    try:
+        a,b,c = sitemap_test(seed_box,None,1)
+    except Exception as e:
+        print (e)
     
 with gr.Blocks() as app:
     file_state=gr.State()