Update app.py

app.py

@@ -1,66 +1,240 @@
-def analyze_character_positions(df):
-    # Character position analysis
-    char_positions = {}
-    char_neighbors = {}
-    
-    for row in df.itertuples():
-        for col in [c for c in df.columns if c.startswith('t')]:
-            word = getattr(row, col)
-            if pd.notna(word) and word != '$':
-                chars = word.strip('"').split(',')
-                for i, char in enumerate(chars):
-                    # Track positions
-                    if char not in char_positions:
-                        char_positions[char] = []
-                    char_positions[char].append(i)
-                    
-                    # Track neighbors
-                    if char not in char_neighbors:
-                        char_neighbors[char] = {'before': [], 'after': []}
-                    if i > 0:
-                        char_neighbors[char]['before'].append(chars[i-1])
-                    if i < len(chars) - 1:
-                        char_neighbors[char]['after'].append(chars[i+1])
-
-def analyze_word_positions(df):
-    # Word position and neighbor analysis
-    word_positions = {}
-    word_neighbors = {}
-    
-    for row in df.itertuples():
+import streamlit as st
+import pandas as pd
+import numpy as np
+import plotly.express as px
+import plotly.graph_objects as go
+import seaborn as sns
+import matplotlib.pyplot as plt
+from collections import defaultdict, Counter
+import base64
+from sklearn.manifold import MDS
+import networkx as nx
+
+# Set page configuration
+st.set_page_config(layout="wide", page_title="Voynich Manuscript Analyzer", page_icon="📜")
+
+# Caching expensive computations
+@st.cache_data
+def load_data(uploaded_file):
+    return pd.read_csv(uploaded_file)
+
+@st.cache_data
+def extract_word_and_chars(token):
+    if token == '$' or '<START>' in token or '<END>' in token:
+        return None, None
+    
+    chars = []
+    temp_chars = token.split(',')
+    
+    for char in temp_chars:
+        if '?' in char:
+            base_char = char.replace('?', '')
+            if base_char:
+                chars.append(base_char)
+            chars.append('?')
+        else:
+            chars.append(char)
+    
+    word = ''.join(chars)
+    return word, chars
+
+@st.cache_data
+def analyze_csv(df):
+    words = []
+    chars_list = []
+    char_positions = defaultdict(list)
+    char_connections = defaultdict(Counter)
+    word_positions = []
+    folio_word_map = defaultdict(Counter)
+    token_columns = [col for col in df.columns if col.startswith('t')]
+    
+    for _, row in df.iterrows():
         line_words = []
-        for col in [c for c in df.columns if c.startswith('t')]:
-            word = getattr(row, col)
-            if pd.notna(word) and word != '$':
-                line_words.append(word)
+        for pos, col in enumerate(token_columns):
+            token = row[col]
+            if pd.notna(token) and token != '$':
+                if token.startswith('"') and token.endswith('"'):
+                    token = token[1:-1]
                 
-        for i, word in enumerate(line_words):
-            # Track positions
-            if word not in word_positions:
-                word_positions[word] = []
-            word_positions[word].append(i)
+                word, chars = extract_word_and_chars(token)
+                if word:
+                    words.append(word)
+                    chars_list.append(chars)
+                    line_words.append((word, pos, chars))
+                    folio_word_map[row['folio']][word] += 1
+                    
+                    for j, char in enumerate(chars):
+                        char_positions[char].append(j)
+                    
+                    for j in range(len(chars) - 1):
+                        char_connections[chars[j]][chars[j+1]] += 1
+        
+        if line_words:
+            word_positions.append({
+                'folio': row['folio'],
+                'par': row['par'],
+                'line': row['line'],
+                'words': line_words
+            })
+    
+    return words, chars_list, char_positions, char_connections, word_positions, folio_word_map
+
+@st.cache_data
+def analyze_trigrams(words, chars_list):
+    char_trigrams = Counter()
+    word_trigrams = Counter()
+    
+    for chars in chars_list:
+        for i in range(len(chars)-2):
+            trigram = tuple(chars[i:i+3])
+            char_trigrams[trigram] += 1
+            
+    for i in range(len(words)-2):
+        trigram = tuple(words[i:i+3])
+        word_trigrams[trigram] += 1
+        
+    return char_trigrams, word_trigrams
+
+@st.cache_data
+def create_12_slot_table(chars_list):
+    slot_frequencies = [Counter() for _ in range(12)]
+    
+    for chars in chars_list:
+        for i, char in enumerate(chars[:12]):
+            slot_frequencies[i][char] += 1
             
-            # Track neighbors
-            if word not in word_neighbors:
-                word_neighbors[word] = {'before': [], 'after': []}
-            if i > 0:
-                word_neighbors[word]['before'].append(line_words[i-1])
-            if i < len(line_words) - 1:
-                word_neighbors[word]['after'].append(line_words[i+1])
-    
-    return word_positions, word_neighbors
-
-def display_analysis(char_positions, char_neighbors, word_positions, word_neighbors):
-    st.subheader("Character Analysis")
-    selected_char = st.selectbox("Select character to analyze", sorted(char_positions.keys()))
-    
-    st.write("Positions:", Counter(char_positions[selected_char]))
-    st.write("Most common previous characters:", Counter(char_neighbors[selected_char]['before']).most_common(5))
-    st.write("Most common following characters:", Counter(char_neighbors[selected_char]['after']).most_common(5))
-    
-    st.subheader("Word Analysis")
-    selected_word = st.selectbox("Select word to analyze", sorted(word_positions.keys()))
-    
-    st.write("Positions in line:", Counter(word_positions[selected_word]))
-    st.write("Most common previous words:", Counter(word_neighbors[selected_word]['before']).most_common(5))
-    st.write("Most common following words:", Counter(word_neighbors[selected_word]['after']).most_common(5))
+    data = []
+    all_chars = sorted(set(char for counter in slot_frequencies for char in counter))
+    
+    for char in all_chars:
+        row = {'Character': char}
+        for i in range(12):
+            row[f'Slot_{i+1}'] = slot_frequencies[i][char]
+        data.append(row)
+        
+    return pd.DataFrame(data)
+
+@st.cache_data
+def analyze_slot_structure(chars_list):
+    slot_contents = defaultdict(Counter)
+    max_slots = 0
+    
+    for chars in chars_list:
+        if len(chars) > max_slots:
+            max_slots = len(chars)
+        
+        for i, char in enumerate(chars):
+            slot_contents[i][char] += 1
+    
+    slot_summary = {}
+    for slot in range(max_slots):
+        if slot in slot_contents:
+            common_chars = slot_contents[slot].most_common(10)
+            slot_summary[slot] = common_chars
+    
+    return slot_summary, max_slots
+
+@st.cache_data
+def create_folio_word_scatter(folio_word_map):
+    all_words = set()
+    for word_counter in folio_word_map.values():
+        all_words.update(word_counter.keys())
+    
+    folios = sorted(folio_word_map.keys())
+    word_freq_matrix = np.zeros((len(folios), len(all_words)))
+    
+    for i, folio in enumerate(folios):
+        for j, word in enumerate(all_words):
+            word_freq_matrix[i, j] = folio_word_map[folio][word]
+    
+    mds = MDS(n_components=2, random_state=42)
+    folio_coords = mds.fit_transform(word_freq_matrix)
+    
+    fig = px.scatter(x=folio_coords[:, 0], y=folio_coords[:, 1], text=folios)
+    fig.update_traces(textposition='top center')
+    fig.update_layout(title='Folio Similarity based on Word Usage',
+                     xaxis_title='Dimension 1',
+                     yaxis_title='Dimension 2')
+    
+    return fig
+
+@st.cache_data
+def plot_char_positions(char_positions, max_slots):
+    chars = []
+    positions = []
+    counts = []
+    
+    for char, pos_list in char_positions.items():
+        pos_counts = Counter(pos_list)
+        for pos, count in pos_counts.items():
+            if pos < max_slots:
+                chars.append(char)
+                positions.append(pos)
+                counts.append(count)
+    
+    df = pd.DataFrame({
+        'Character': chars,
+        'Position': positions,
+        'Count': counts
+    })
+    
+    pivot_df = df.pivot(index='Character', columns='Position', values='Count').fillna(0)
+    
+    fig = px.imshow(pivot_df, color_continuous_scale='YlGnBu', labels=dict(x="Position in Word", y="Character", color="Frequency"))
+    fig.update_layout(title='Character Position Heatmap')
+    return fig
+
+@st.cache_data
+def get_download_link_csv(df, filename):
+    csv = df.to_csv(index=False)
+    b64 = base64.b64encode(csv.encode()).decode()
+    href = f'<a href="data:file/csv;base64,{b64}" download="{filename}">Download CSV</a>'
+    return href
+
+# Main App
+st.title("Voynich Manuscript Analyzer")
+st.write("Upload your CSV file to discover potential patterns and character distributions.")
+
+uploaded_file = st.file_uploader("Choose a CSV file", type="csv")
+
+if uploaded_file is not None:
+    df = load_data(uploaded_file)
+    words, chars_list, char_positions, char_connections, word_positions, folio_word_map = analyze_csv(df)
+    
+    with st.expander("Basic Statistics"):
+        st.write(f"Total words: {len(words)}")
+        st.write(f"Total unique words: {len(set(words))}")
+        unique_chars = set(char for chars in chars_list for char in chars)
+        st.write(f"Total unique characters: {len(unique_chars)}")
+        st.write("Unique characters:", ", ".join(sorted(unique_chars)))
+    
+    with st.expander("Trigram Analysis"):
+        char_trigrams, word_trigrams = analyze_trigrams(words, chars_list)
+        
+        st.write("Top 20 Character Trigrams")
+        char_trigram_df = pd.DataFrame([
+            {'Trigram': ' - '.join(trigram), 'Count': count}
+            for trigram, count in char_trigrams.most_common(20)
+        ])
+        st.dataframe(char_trigram_df)
+        st.markdown(get_download_link_csv(char_trigram_df, "char_trigrams.csv"), unsafe_allow_html=True)
+        
+        st.write("Top 20 Word Trigrams")
+        word_trigram_df = pd.DataFrame([
+            {'Trigram': ' - '.join(trigram), 'Count': count}
+            for trigram, count in word_trigrams.most_common(20)
+        ])
+        st.dataframe(word_trigram_df)
+        st.markdown(get_download_link_csv(word_trigram_df, "word_trigrams.csv"), unsafe_allow_html=True)
+
+    with st.expander("Character Position Analysis"):
+        slot_summary, max_slots = analyze_slot_structure(chars_list)
+        st.plotly_chart(plot_char_positions(char_positions, max_slots))
+
+    with st.expander("Folio Similarity Analysis"):
+        st.plotly_chart(create_folio_word_scatter(folio_word_map))
+
+    with st.expander("12-Slot Character Frequency Table"):
+        slot_freq_df = create_12_slot_table(chars_list)
+        st.dataframe(slot_freq_df)
+        st.markdown(get_download_link_csv(slot_freq_df, "slot_frequencies.csv"), unsafe_allow_html=True)