Update app.py

app.py

@@ -15,7 +15,9 @@ api=HfApi(token="")
 filename="urls"
 filename2="pages"
 
-def init():
+def init(filename=None):
+    if filename==None:
+        filename=filename
     r = requests.get(f'{save_data}crawl/{filename}.json') 
     print(f'status code main:: {r.status_code}')
     if r.status_code==200:
@@ -335,14 +337,22 @@ def sitemap(url,file_state,level):
                                 print (e)
                 except Exception as e:
                     print (e)
-        #url_page=[]
+        '''url_page=[]
         url_front=[]
+        url_json=[]
         for ea_link in link2['TREE']:
             url_list=ea_link['URL'].split("/")
             url_front.append("".join(x for x in url_list[:3]))
+            url_page.append("/".join(z for z in url_list[3:]))
             print(f'URL_FRONT:: {url_front}')
         #url_key=sort
-        uri_key=sort_doc(url_front,file_state,8)
+        for each_link in uri_key.keys():
+            out_file=init(f'{each_link}.json')
+        
+        '''
+            
+        uri_key=sort_doc(link2['TREE'],file_state,8)
+            
 ######## Save Database ########
     uid=uuid.uuid4()
     #for ea in list(uri_key.keys()):