import gradio as gr import pandas as pd import networkx as nx from pyvis.network import Network import tempfile import os import re from collections import Counter # A robust built-in stopword list to guarantee functionality even without downloading NLTK datasets DEFAULT_STOPWORDS = set([ "i", "me", "my", "myself", "we", "our", "ours", "ourselves", "you", "your", "yours", "yourself", "yourselves", "he", "him", "his", "himself", "she", "her", "hers", "herself", "it", "its", "itself", "they", "them", "their", "theirs", "themselves", "what", "which", "who", "whom", "this", "that", "these", "those", "am", "is", "are", "was", "were", "be", "been", "being", "have", "has", "had", "having", "do", "does", "did", "doing", "a", "an", "the", "and", "but", "if", "or", "because", "as", "until", "while", "of", "at", "by", "for", "with", "about", "against", "between", "into", "through", "during", "before", "after", "above", "below", "to", "from", "up", "down", "in", "out", "on", "off", "over", "under", "again", "further", "then", "once", "here", "there", "when", "where", "why", "how", "all", "any", "both", "each", "few", "more", "most", "other", "some", "such", "no", "nor", "not", "only", "own", "same", "so", "than", "too", "very", "s", "t", "can", "will", "just", "don", "should", "now", "d", "ll", "m", "o", "re", "ve", "y", "ain", "aren", "couldn", "didn", "doesn", "hadn", "hasn", "haven", "isn", "ma", "mightn", "mustn", "needn", "shan", "shouldn", "wasn", "weren", "won", "wouldn" ]) def tokenize_and_clean(text, custom_stopwords_str): # Convert custom stopwords custom_stops = set([w.strip().lower() for w in custom_stopwords_str.split(',') if w.strip()]) all_stopwords = DEFAULT_STOPWORDS.union(custom_stops) # Simple regex tokenizer words = re.findall(r'\b[a-zA-Z]{2,}\b', text.lower()) cleaned_words = [w for w in words if w not in all_stopwords] return cleaned_words def build_cooccurrence_matrix(words, window_size): cooccurrences = Counter() for i in range(len(words)): current_word = words[i] # Look ahead up to the window size start = i + 1 end = min(i + 1 + window_size, len(words)) for j in range(start, end): next_word = words[j] if current_word != next_word: # Sort alphabetically to keep edges undirected pair = tuple(sorted([current_word, next_word])) cooccurrences[pair] += 1 return cooccurrences def generate_vis_html(cooccurrences, min_freq, word_counts): # Filter by minimum frequency filtered_pairs = {pair: count for pair, count in cooccurrences.items() if count >= min_freq} if not filtered_pairs: return None, None # Extract unique nodes from filtered edges nodes = set() for pair in filtered_pairs.keys(): nodes.update(pair) # Initialize PyVis Network net = Network( height="500px", width="100%", bgcolor="#16100c", font_color="#f4eee6", notebook=False ) net.set_options(""" var options = { "nodes": { "borderWidth": 1, "borderWidthSelected": 3, "color": { "border": "#2c1e16", "background": "#ff7043", "highlight": { "border": "#ff7043", "background": "#ffffff" } }, "font": { "color": "#f4eee6", "size": 14, "face": "Inter, sans-serif" } }, "edges": { "color": { "color": "rgba(255, 112, 67, 0.3)", "highlight": "#ff7043" }, "smooth": { "type": "continuous" } }, "physics": { "barnesHut": { "gravitationalConstant": -15000, "centralGravity": 0.35, "springLength": 100, "springConstant": 0.05 }, "minVelocity": 0.75 } } """) # Add nodes (scaled by absolute frequency in the text) for node in nodes: freq = word_counts.get(node, 1) size = 10 + min(freq * 1.5, 40) # Caps size to prevent massive nodes net.add_node(node, label=node, size=size, title=f"Word occurrences: {freq}") # Add edges for (source, target), weight in filtered_pairs.items(): net.add_edge(source, target, value=weight, title=f"Co-occurrences: {weight}") # Save graph and read temp_dir = tempfile.gettempdir() temp_path = os.path.join(temp_dir, next(tempfile._get_candidate_names()) + ".html") net.save_graph(temp_path) with open(temp_path, "r", encoding="utf-8") as f: html_content = f.read() try: os.remove(temp_path) except: pass escaped_html = html_content.replace('"', '"') iframe_code = f'' # Create DataFrame for display/download edge_list = [{"Source": k[0], "Target": k[1], "Co-occurrences": v} for k, v in filtered_pairs.items()] df = pd.DataFrame(edge_list).sort_values("Co-occurrences", ascending=False) return iframe_code, df def analyze_cooccurrence(text, window_size, min_freq, custom_stopwords): if not text or len(text.strip()) < 10: return "Please input a longer block of text.", None, None, None words = tokenize_and_clean(text, custom_stopwords) if len(words) < 5: return "Not enough meaningful words found after filtering stopwords.", None, None, None word_counts = Counter(words) cooccurrences = build_cooccurrence_matrix(words, window_size) vis_html, df = generate_vis_html(cooccurrences, min_freq, word_counts) if vis_html is None: return f"No word pairs met the minimum co-occurrence threshold of {min_freq}. Try lowering the slider.", None, None, None # Stats num_nodes = len(df['Source'].unique()) + len(df['Target'].unique()) num_edges = len(df) stats_html = f"""
Words in Network
{num_nodes}
Unique Word Pairs (Links)
{num_edges}
""" # Download file path out_csv = tempfile.mktemp(suffix=".csv") df.to_csv(out_csv, index=False) return "", stats_html, vis_html, df.head(50), out_csv theme = gr.themes.Default( primary_hue="orange", neutral_hue="stone" ).set( body_background_fill="#0d0907", body_text_color="#c4bbae", block_background_fill="#16100c", block_border_width="1px", block_label_text_color="#f4eee6" ) with gr.Blocks(theme=theme, title="Word Co-occurrence Networks") as demo: gr.Markdown( """ # 📊 Interactive Word Co-occurrence Networks ### Map how words and concepts interconnect based on proximity. Perfect for structural linguistics, narrative themes, and semantic modeling. """ ) error_msg = gr.Markdown("", visible=False) with gr.Row(): with gr.Column(scale=1): raw_text = gr.Textbox( label="Input Text Document", placeholder="Paste your textual content here (e.g. speeches, novel chapters, or code blocks)...", lines=10 ) with gr.Row(): window_size = gr.Slider( minimum=2, maximum=10, value=5, step=1, label="Sliding Window Size", info="Maximum distance in words to capture a connection." ) min_freq = gr.Slider( minimum=1, maximum=20, value=3, step=1, label="Min Co-occurrence Frequency", info="Filters out peripheral connections." ) custom_stopwords = gr.Textbox( label="Custom Stopwords (comma separated)", placeholder="chapter, page, said, would", info="Words to ignore during co-occurrence analysis." ) btn = gr.Button("Build Co-occurrence Network", variant="primary") with gr.Column(scale=2): stats_box = gr.HTML() with gr.Tabs(): with gr.TabItem("Interactive Graph"): vis_box = gr.HTML() with gr.TabItem("Data Table"): table_box = gr.Dataframe(headers=["Source", "Target", "Co-occurrences"]) download_btn = gr.File(label="Download Full Dataset") def process(text, window, freq, stops): err, stats, vis, table, csv_path = analyze_cooccurrence(text, window, freq, stops) if err: return gr.update(value=err, visible=True), "", "", None, gr.update(visible=False) return gr.update(visible=False), stats, vis, table, gr.update(value=csv_path, visible=True) btn.click( process, inputs=[raw_text, window_size, min_freq, custom_stopwords], outputs=[error_msg, stats_box, vis_box, table_box, download_btn] ) if __name__ == "__main__": demo.launch()