Spaces:
Runtime error
Runtime error
File size: 3,475 Bytes
16d62e2 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 |
from bs4 import BeautifulSoup, Tag
import datetime
import requests
import re
Url = 'https://huggingface.co'
NoisePatterns = {
'(No)Script': r'<[ ]*(script|noscript)[^>]*?>.*?<\/[ ]*\1[ ]*>',
'Style': r'<[ ]*(style)[^>]*?>.*?<\/[ ]*\1[ ]*>',
'Svg': r'<[ ]*(svg)[^>]*?>.*?<\/[ ]*\1[ ]*>',
'Meta+Link': r'<[ ]*(meta|link)[^>]*?[\/]?[ ]*>',
'Comment': r'<[ ]*!--.*?--[ ]*>',
'Base64Img': r'<[ ]*img[^>]+src="data:image\/[^;]+;base64,[^"]+"[^>]*[\/]?[ ]*>',
'DocType': r'<!(DOCTYPE|doctype)[ ]*[a-z]*>',
'DataAttributes': r'[ ]+data-[\w-]+="[^"]*"',
'Classes': r'[ ]+class="[^"]*"',
'EmptyAttributes': r'[ ]+[a-z-]+=""',
'DateTime': r'[ ]+datetime="[^"]*"',
'EmptyTags': r'(?:<[ ]*([a-z]{1,10})[^>]*>[ \t\r\n]*){1,5}(?:<\/[ ]*\1[ ]*>){1,5}',
'EmptyLines': r'^[ \t]*\r?\n',
}
def RemoveNoise(RawHtml: str) -> str:
'''Remove noise from HTML content.
Args:
RawHtml (str): The raw HTML content.
Returns:
str: Cleaned HTML content without noise.
'''
CleanedHtml = RawHtml
OriginalCharCount = len(RawHtml)
for PatternName, Pattern in NoisePatterns.items():
if PatternName in ['EmptyLines', 'EmptyTags']: # These patterns are line-based
CleanedHtml = re.sub(Pattern, '', CleanedHtml, flags=re.MULTILINE)
else:
CleanedHtml = re.sub(Pattern, '', CleanedHtml, flags=re.DOTALL | re.IGNORECASE | re.MULTILINE)
print(f'• Removed {PatternName} noise. Removed {OriginalCharCount - len(CleanedHtml)} characters.')
OriginalCharCount = len(CleanedHtml)
return CleanedHtml
def FetchHtmlContent(Url: str) -> str | int:
'''Fetch HTML content from a URL.
Args:
Url (str): The URL to fetch HTML content from.
Returns:
str: The raw HTML content.
'''
Headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'
}
Response = requests.get(Url, headers=Headers)
if Response.status_code == 200:
return Response.text
else:
return Response.status_code
Start = datetime.datetime.now()
RawHtml = FetchHtmlContent(Url)
if isinstance(RawHtml, str):
RawCharCount = len(RawHtml)
print('Prettifying HTML content...')
Soup = BeautifulSoup(RawHtml, 'html.parser')
PrettifiedHtml = str(Soup.prettify())
Title = Soup.title.string if Soup.title else 'No title found'
MetaDesc = Soup.find('meta', attrs={'name': 'description'})
Description = MetaDesc.get('content', 'No description found') if isinstance(MetaDesc, Tag) else 'No description found'
print('Purifying HTML content...')
CleanedHtml = RemoveNoise(PrettifiedHtml)
CleanedCharCount = len(CleanedHtml)
Ratio = CleanedCharCount / RawCharCount if RawCharCount > 0 else 0
Summary = [
'<!-- --- Purification Summary ---',
f'URL: {Url}',
f'Title: {Title}',
f'Description: {Description}',
f'Time of Fetch: {datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")} (Took {datetime.datetime.now() - Start})',
f'Noise Removal Ratio: {Ratio:.2%} (lower is better)',
f'Characters: {RawCharCount} -> {CleanedCharCount} ({RawCharCount - CleanedCharCount} characters removed)',
'----------------------------- -->'
]
for Line in Summary:
print(Line)
with open('CleanedHtml.html', 'w', encoding='utf-8') as File:
for Line in Summary:
File.write(Line + '\n')
File.write(CleanedHtml)
else:
print(f'Failed to fetch HTML content. Status code: {RawHtml}') |