app.py

import gradio as gr
import pandas as pd
import networkx as nx
from pyvis.network import Network
import tempfile
import os
import re
from collections import Counter

# A robust built-in stopword list to guarantee functionality even without downloading NLTK datasets
DEFAULT_STOPWORDS = set([
    "i", "me", "my", "myself", "we", "our", "ours", "ourselves", "you", "your", "yours", 
    "yourself", "yourselves", "he", "him", "his", "himself", "she", "her", "hers", 
    "herself", "it", "its", "itself", "they", "them", "their", "theirs", "themselves", 
    "what", "which", "who", "whom", "this", "that", "these", "those", "am", "is", "are", 
    "was", "were", "be", "been", "being", "have", "has", "had", "having", "do", "does", 
    "did", "doing", "a", "an", "the", "and", "but", "if", "or", "because", "as", "until", 
    "while", "of", "at", "by", "for", "with", "about", "against", "between", "into", 
    "through", "during", "before", "after", "above", "below", "to", "from", "up", "down", 
    "in", "out", "on", "off", "over", "under", "again", "further", "then", "once", "here", 
    "there", "when", "where", "why", "how", "all", "any", "both", "each", "few", "more", 
    "most", "other", "some", "such", "no", "nor", "not", "only", "own", "same", "so", 
    "than", "too", "very", "s", "t", "can", "will", "just", "don", "should", "now", "d", 
    "ll", "m", "o", "re", "ve", "y", "ain", "aren", "couldn", "didn", "doesn", "hadn", 
    "hasn", "haven", "isn", "ma", "mightn", "mustn", "needn", "shan", "shouldn", "wasn", 
    "weren", "won", "wouldn"
])

def tokenize_and_clean(text, custom_stopwords_str):
    # Convert custom stopwords
    custom_stops = set([w.strip().lower() for w in custom_stopwords_str.split(',') if w.strip()])
    all_stopwords = DEFAULT_STOPWORDS.union(custom_stops)
    
    # Simple regex tokenizer
    words = re.findall(r'\b[a-zA-Z]{2,}\b', text.lower())
    cleaned_words = [w for w in words if w not in all_stopwords]
    return cleaned_words

def build_cooccurrence_matrix(words, window_size):
    cooccurrences = Counter()
    
    for i in range(len(words)):
        current_word = words[i]
        # Look ahead up to the window size
        start = i + 1
        end = min(i + 1 + window_size, len(words))
        
        for j in range(start, end):
            next_word = words[j]
            if current_word != next_word:
                # Sort alphabetically to keep edges undirected
                pair = tuple(sorted([current_word, next_word]))
                cooccurrences[pair] += 1
                
    return cooccurrences

def generate_vis_html(cooccurrences, min_freq, word_counts):
    # Filter by minimum frequency
    filtered_pairs = {pair: count for pair, count in cooccurrences.items() if count >= min_freq}
    
    if not filtered_pairs:
        return None, None
        
    # Extract unique nodes from filtered edges
    nodes = set()
    for pair in filtered_pairs.keys():
        nodes.update(pair)
        
    # Initialize PyVis Network
    net = Network(
        height="500px", 
        width="100%", 
        bgcolor="#16100c", 
        font_color="#f4eee6", 
        notebook=False
    )
    
    net.set_options("""
    var options = {
      "nodes": {
        "borderWidth": 1,
        "borderWidthSelected": 3,
        "color": {
          "border": "#2c1e16",
          "background": "#ff7043",
          "highlight": {
            "border": "#ff7043",
            "background": "#ffffff"
          }
        },
        "font": {
          "color": "#f4eee6",
          "size": 14,
          "face": "Inter, sans-serif"
        }
      },
      "edges": {
        "color": {
          "color": "rgba(255, 112, 67, 0.3)",
          "highlight": "#ff7043"
        },
        "smooth": {
          "type": "continuous"
        }
      },
      "physics": {
        "barnesHut": {
          "gravitationalConstant": -15000,
          "centralGravity": 0.35,
          "springLength": 100,
          "springConstant": 0.05
        },
        "minVelocity": 0.75
      }
    }
    """)
    
    # Add nodes (scaled by absolute frequency in the text)
    for node in nodes:
        freq = word_counts.get(node, 1)
        size = 10 + min(freq * 1.5, 40)  # Caps size to prevent massive nodes
        net.add_node(node, label=node, size=size, title=f"Word occurrences: {freq}")
        
    # Add edges
    for (source, target), weight in filtered_pairs.items():
        net.add_edge(source, target, value=weight, title=f"Co-occurrences: {weight}")
        
    # Save graph and read
    temp_dir = tempfile.gettempdir()
    temp_path = os.path.join(temp_dir, next(tempfile._get_candidate_names()) + ".html")
    net.save_graph(temp_path)
    
    with open(temp_path, "r", encoding="utf-8") as f:
        html_content = f.read()
        
    try:
        os.remove(temp_path)
    except:
        pass
        
    escaped_html = html_content.replace('"', '"')
    iframe_code = f'<iframe srcdoc="{escaped_html}" style="width: 100%; height: 530px; border: 1px solid rgba(255, 255, 255, 0.08); border-radius: 8px;"></iframe>'
    
    # Create DataFrame for display/download
    edge_list = [{"Source": k[0], "Target": k[1], "Co-occurrences": v} for k, v in filtered_pairs.items()]
    df = pd.DataFrame(edge_list).sort_values("Co-occurrences", ascending=False)
    
    return iframe_code, df

def analyze_cooccurrence(text, window_size, min_freq, custom_stopwords):
    if not text or len(text.strip()) < 10:
        return "Please input a longer block of text.", None, None, None
        
    words = tokenize_and_clean(text, custom_stopwords)
    if len(words) < 5:
        return "Not enough meaningful words found after filtering stopwords.", None, None, None
        
    word_counts = Counter(words)
    cooccurrences = build_cooccurrence_matrix(words, window_size)
    
    vis_html, df = generate_vis_html(cooccurrences, min_freq, word_counts)
    
    if vis_html is None:
        return f"No word pairs met the minimum co-occurrence threshold of {min_freq}. Try lowering the slider.", None, None, None
        
    # Stats
    num_nodes = len(df['Source'].unique()) + len(df['Target'].unique())
    num_edges = len(df)
    
    stats_html = f"""
    <div style='display: grid; grid-template-columns: repeat(2, 1fr); gap: 1rem; margin-bottom: 1rem;'>
        <div style='background: rgba(255, 255, 255, 0.03); border: 1px solid rgba(255, 255, 255, 0.08); border-radius: 8px; padding: 1rem; text-align: center;'>
            <div style='font-size: 0.75rem; text-transform: uppercase; color: #ff7043; letter-spacing: 0.1em;'>Words in Network</div>
            <div style='font-size: 2rem; font-weight: bold; margin-top: 0.5rem;'>{num_nodes}</div>
        </div>
        <div style='background: rgba(255, 255, 255, 0.03); border: 1px solid rgba(255, 255, 255, 0.08); border-radius: 8px; padding: 1rem; text-align: center;'>
            <div style='font-size: 0.75rem; text-transform: uppercase; color: #ff7043; letter-spacing: 0.1em;'>Unique Word Pairs (Links)</div>
            <div style='font-size: 2rem; font-weight: bold; margin-top: 0.5rem;'>{num_edges}</div>
        </div>
    </div>
    """
    
    # Download file path
    out_csv = tempfile.mktemp(suffix=".csv")
    df.to_csv(out_csv, index=False)
    
    return "", stats_html, vis_html, df.head(50), out_csv

theme = gr.themes.Default(
    primary_hue="orange",
    neutral_hue="stone"
).set(
    body_background_fill="#0d0907",
    body_text_color="#c4bbae",
    block_background_fill="#16100c",
    block_border_width="1px",
    block_label_text_color="#f4eee6"
)

with gr.Blocks(theme=theme, title="Word Co-occurrence Networks") as demo:
    gr.Markdown(
        """
        # 📊 Interactive Word Co-occurrence Networks
        ### Map how words and concepts interconnect based on proximity. Perfect for structural linguistics, narrative themes, and semantic modeling.
        """
    )
    
    error_msg = gr.Markdown("", visible=False)
    
    with gr.Row():
        with gr.Column(scale=1):
            raw_text = gr.Textbox(
                label="Input Text Document", 
                placeholder="Paste your textual content here (e.g. speeches, novel chapters, or code blocks)...", 
                lines=10
            )
            
            with gr.Row():
                window_size = gr.Slider(
                    minimum=2, 
                    maximum=10, 
                    value=5, 
                    step=1, 
                    label="Sliding Window Size",
                    info="Maximum distance in words to capture a connection."
                )
                min_freq = gr.Slider(
                    minimum=1, 
                    maximum=20, 
                    value=3, 
                    step=1, 
                    label="Min Co-occurrence Frequency",
                    info="Filters out peripheral connections."
                )
                
            custom_stopwords = gr.Textbox(
                label="Custom Stopwords (comma separated)",
                placeholder="chapter, page, said, would",
                info="Words to ignore during co-occurrence analysis."
            )
            
            btn = gr.Button("Build Co-occurrence Network", variant="primary")
            
        with gr.Column(scale=2):
            stats_box = gr.HTML()
            
            with gr.Tabs():
                with gr.TabItem("Interactive Graph"):
                    vis_box = gr.HTML()
                with gr.TabItem("Data Table"):
                    table_box = gr.Dataframe(headers=["Source", "Target", "Co-occurrences"])
                    download_btn = gr.File(label="Download Full Dataset")

    def process(text, window, freq, stops):
        err, stats, vis, table, csv_path = analyze_cooccurrence(text, window, freq, stops)
        if err:
            return gr.update(value=err, visible=True), "", "", None, gr.update(visible=False)
        return gr.update(visible=False), stats, vis, table, gr.update(value=csv_path, visible=True)

    btn.click(
        process,
        inputs=[raw_text, window_size, min_freq, custom_stopwords],
        outputs=[error_msg, stats_box, vis_box, table_box, download_btn]
    )

if __name__ == "__main__":
    demo.launch()