Update app.py
Browse files
app.py
CHANGED
|
@@ -270,31 +270,34 @@ def link_find(url):
|
|
| 270 |
q=("a","p","span","content","article")
|
| 271 |
for p in soup.find_all("a"):
|
| 272 |
url0=p.get('href')
|
| 273 |
-
|
| 274 |
-
|
| 275 |
-
|
| 276 |
-
|
| 277 |
-
|
| 278 |
-
|
| 279 |
-
|
| 280 |
-
|
| 281 |
-
|
| 282 |
-
|
| 283 |
-
|
| 284 |
-
|
| 285 |
-
|
| 286 |
-
|
| 287 |
-
|
| 288 |
-
|
| 289 |
-
|
| 290 |
-
|
| 291 |
-
|
| 292 |
-
|
| 293 |
-
|
| 294 |
-
|
| 295 |
-
|
| 296 |
-
|
| 297 |
-
|
|
|
|
|
|
|
|
|
|
| 298 |
else:
|
| 299 |
print("NO")
|
| 300 |
pass
|
|
|
|
| 270 |
q=("a","p","span","content","article")
|
| 271 |
for p in soup.find_all("a"):
|
| 272 |
url0=p.get('href')
|
| 273 |
+
try:
|
| 274 |
+
if url0.startswith("//"):
|
| 275 |
+
print(url0)
|
| 276 |
+
uri1=url.split("//")[0]
|
| 277 |
+
#uri2=url.split("//")[1]
|
| 278 |
+
#uri3=uri2.split("/")[0]
|
| 279 |
+
#uri=f'{uri1}//{uri3}'
|
| 280 |
+
uri=f'{uri1}{url0}'
|
| 281 |
+
#print(uri)
|
| 282 |
+
elif url0.startswith("/") and not url0.startswith("//"):
|
| 283 |
+
uri1=url.split("//")[0]
|
| 284 |
+
uri2=url.split("//")[1]
|
| 285 |
+
uri3=uri2.split("/")[0]
|
| 286 |
+
uri=f'{uri1}//{uri3}'
|
| 287 |
+
uri=f'{uri}{url0}'
|
| 288 |
+
#print(uri)
|
| 289 |
+
else:
|
| 290 |
+
uri=url0
|
| 291 |
+
|
| 292 |
+
node1['LINKS'].append(uri)
|
| 293 |
+
node1['TREE'].append({"URL":uri,"TITLE":p.get('title'),"STRING":p.string,"TEXT":"","LINKS":[],"TREE":[]})
|
| 294 |
+
node2['TREE'].append({"URL":uri,"LINKS":[],"TREE":[]})
|
| 295 |
+
node2['LINKS'].append(uri)
|
| 296 |
+
#node2['LINK_KEY'].append(uri_key)
|
| 297 |
+
link_box.append(uri)
|
| 298 |
+
#out.append({"URL":p.get('href'),"TITLE":p.get('title'),"STRING":p.string,"TEXT":"","TREE":[]})
|
| 299 |
+
except Exception as e:
|
| 300 |
+
print (e)
|
| 301 |
else:
|
| 302 |
print("NO")
|
| 303 |
pass
|