Spaces:
Runtime error
Runtime error
| import re | |
| from collections import Counter | |
| from typing import List, Tuple | |
| import gradio as gr | |
| # --- NLTK setup --- | |
| import nltk | |
| from nltk.tokenize import word_tokenize, sent_tokenize | |
| STOPWORDS = set() | |
| PUNCT_RE = re.compile(r"[^\w\s]") # remove punctuation (keeps letters, digits, underscore, and spaces) | |
| def ensure_nltk(): | |
| """Download required NLTK resources and load stopwords.""" | |
| global STOPWORDS | |
| try: | |
| nltk.download("punkt") | |
| nltk.download("stopwords") | |
| from nltk.corpus import stopwords | |
| STOPWORDS = set(stopwords.words("english")) | |
| return True, "NLTK data installed (punkt, stopwords)." | |
| except Exception as e: | |
| return False, f"NLTK install error: {e}" | |
| def read_text_area_or_file(f, text): | |
| """Priority: uploaded file if present, else textbox. Supports .txt and .docx""" | |
| if f is not None: | |
| path = getattr(f, "name", None) | |
| if path and path.lower().endswith(".txt"): | |
| with open(path, "r", encoding="utf-8", errors="ignore") as fh: | |
| return fh.read() | |
| elif path and path.lower().endswith(".docx"): | |
| import docx2txt | |
| return docx2txt.process(path) or "" | |
| else: | |
| return f"Unsupported file type. Please upload .txt or .docx." | |
| return text or "" | |
| def normalize_and_tokenize(text: str) -> Tuple[List[str], List[List[str]]]: | |
| """Lowercase, remove punctuation, drop stopwords, return sentences + tokenized sentences""" | |
| lowered = text.lower() | |
| no_punct = PUNCT_RE.sub(" ", lowered) | |
| sentences = [s.strip() for s in sent_tokenize(no_punct) if s.strip()] | |
| if not sentences: | |
| rough = re.split(r"[.\n]+", no_punct) | |
| sentences = [s.strip() for s in rough if s.strip()] | |
| tokenized = [] | |
| for s in sentences: | |
| toks = [t for t in word_tokenize(s) if t and t not in STOPWORDS and not t.isnumeric()] | |
| tokenized.append(toks) | |
| return sentences, tokenized | |
| def build_bow(tokenized_sentences: List[List[str]]) -> Counter: | |
| all_words = [w for sent in tokenized_sentences for w in sent] | |
| return Counter(all_words) | |
| def vectorize_sentence(tokenized_sentences: List[List[str]], sentence_index: int, vocabulary: List[str]) -> List[int]: | |
| if not tokenized_sentences: | |
| return [] | |
| idx = max(0, min(sentence_index, len(tokenized_sentences) - 1)) | |
| sent_tokens = tokenized_sentences[idx] | |
| return [sent_tokens.count(word) for word in vocabulary] | |
| def process(menu_choice, text, file, sentence_index): | |
| if menu_choice == "Install NLTK": | |
| ok, msg = ensure_nltk() | |
| return gr.update(value=msg), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False) | |
| raw = read_text_area_or_file(file, text) | |
| if not raw or raw.startswith("Unsupported file type"): | |
| return gr.update(value=raw or "No input text found."), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False) | |
| sentences, tokenized = normalize_and_tokenize(raw) | |
| if menu_choice == "Tokenize sentences into words": | |
| lines = [f"Sentence {i+1}: {toks}" for i, toks in enumerate(tokenized)] | |
| return gr.update(value="\n".join(lines)), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False) | |
| bow = build_bow(tokenized) | |
| if menu_choice == "Count word occurrences (Bag of Words)": | |
| items = sorted(bow.items(), key=lambda x: (-x[1], x[0])) | |
| text_out = "Bag of Words (word -> count):\n" + "\n".join([f"{w}: {c}" for w, c in items]) | |
| return gr.update(value=text_out), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False) | |
| if menu_choice == "Build a word frequency vector for any selected sentence": | |
| vocab = sorted(bow.keys()) | |
| vec = vectorize_sentence(tokenized, sentence_index, vocab) | |
| chosen_idx = max(0, min(sentence_index, len(tokenized) - 1)) | |
| chosen_sentence = " ".join(sentences[chosen_idx].split()) | |
| text_summary = ( | |
| f"Selected sentence index: {chosen_idx} (1-based: {chosen_idx+1})\n" | |
| f"Sentence tokens: {tokenized[chosen_idx]}\n" | |
| f"Vocabulary size: {len(vocab)}\n" | |
| f"Vector length: {len(vec)}" | |
| ) | |
| vocab_text = ", ".join(vocab) if vocab else "(empty vocabulary)" | |
| vector_text = ", ".join(map(str, vec)) if vec else "(empty vector)" | |
| return ( | |
| gr.update(value=text_summary + f"\n\nSentence (cleaned): {chosen_sentence}"), | |
| gr.update(value=vocab_text, visible=True, label="Vocabulary (ordered)"), | |
| gr.update(value=vector_text, visible=True, label="Word Frequency Vector (aligned with vocabulary)"), | |
| gr.update(visible=True) | |
| ) | |
| return gr.update(value="Choose a menu option."), gr.update(visible=False), gr.update(visible=False), gr.update(visible=False) | |
| with gr.Blocks(title="BoW + Vectorizer (NLTK + Gradio)") as demo: | |
| gr.Markdown("# NLP Utilities: Tokenize • Bag of Words • Sentence Vector\nText is lowercased, punctuation removed, and stopwords dropped.") | |
| menu = gr.Radio( | |
| choices=[ | |
| "Install NLTK", | |
| "Tokenize sentences into words", | |
| "Count word occurrences (Bag of Words)", | |
| "Build a word frequency vector for any selected sentence", | |
| ], | |
| value="Tokenize sentences into words", | |
| label="Menu" | |
| ) | |
| text_in = gr.Textbox(lines=8, label="Text", placeholder="Paste text here...") | |
| file_in = gr.File(label="Or drop a .txt / .docx file", file_types=[".txt", ".docx"]) | |
| sentence_index = gr.Number(value=0, precision=0, visible=False, label="Sentence index (0-based)") | |
| run_btn = gr.Button("Process", variant="primary") | |
| out_main = gr.Textbox(label="Output", lines=12) | |
| out_vocab = gr.Textbox(label="Vocabulary (ordered)", lines=6, visible=False) | |
| out_vector = gr.Textbox(label="Word Frequency Vector (aligned with vocabulary)", lines=6, visible=False) | |
| run_btn.click(fn=process, inputs=[menu, text_in, file_in, sentence_index], outputs=[out_main, out_vocab, out_vector, sentence_index]) | |
| if __name__ == "__main__": | |
| # safe: pre-install NLTK resources at startup | |
| ensure_nltk() | |
| demo.launch() | |