Spaces:
Sleeping
Sleeping
File size: 1,836 Bytes
c8b1fd7 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 | import re
from bs4 import BeautifulSoup
import spacy
from collections import Counter
try:
nlp = spacy.load("en_core_web_sm")
except OSError:
raise OSError("spaCy model not found. Run: python -m spacy download en_core_web_sm")
def remove_html(text: str )-> str:
"""Removes HTML tags from text"""
soup = BeautifulSoup(text, "html.parser")
return soup.get_text(separator=" ")
def clean_text(text: str) -> str:
"""Cleans and noramlizes text."""
text = remove_html(text)
text = text.strip()
text = re.sub(r"\s+", " ", text)
text = re.sub(r"([.!?])\1+", r"\1", text)
return text
def extract_links(text: str)-> list:
"""Extract URLs from text."""
pattern = r"https?://\S+"
return re.findall(pattern, text)
def extract_keywords(text: str, top_n: int =10) ->list:
"""Returns impotant keywords"""
doc = nlp(text)
words =[]
for token in doc:
if ( token.is_stop or token.is_punct or token.is_space):
continue
if len(token.text) <=2:
continue
words.append(token.lemma_.lower())
frequency= Counter(words)
keywords = frequency.most_common(top_n)
return [ word for word, count in keywords]
def get_word_count(text:str)-> int:
return len(text.split())
def get_sentence_count(text:str)-> int:
sentences = re.split(r"[.!?]+", text)
sentences = [sentence for sentence in sentences if sentence.strip()]
return len(sentences)
def parse_text(text: str) -> dict:
"""Main preprocessing function."""
cleaned_text = clean_text(text)
return {
"clean_text": cleaned_text,
"keywords": extract_keywords(cleaned_text),
"links": extract_links(cleaned_text),
"word_count": get_word_count(cleaned_text),
"sentence_count": get_sentence_count(cleaned_text)
}
|