Rename text_analyzer.py to data_analysis.py

data_analysis.py

@@ -0,0 +1,41 @@
+import pandas as pd
+import numpy as np
+from scipy import stats
+
+class DataAnalyzer:
+    def analyze(self, data):
+        insights = {}
+
+        # Basic statistics
+        insights['basic_stats'] = data.describe().to_dict()
+
+        # Correlation analysis
+        numeric_columns = data.select_dtypes(include=[np.number]).columns
+        if len(numeric_columns) > 1:
+            correlation_matrix = data[numeric_columns].corr()
+            insights['correlations'] = correlation_matrix.to_dict()
+
+        # Skewness and kurtosis
+        skewness = data[numeric_columns].skew()
+        kurtosis = data[numeric_columns].kurtosis()
+        insights['distribution'] = {
+            'skewness': skewness.to_dict(),
+            'kurtosis': kurtosis.to_dict()
+        }
+
+        # Categorical data analysis
+        categorical_columns = data.select_dtypes(include=['object']).columns
+        for column in categorical_columns:
+            insights[f'{column}_distribution'] = data[column].value_counts().to_dict()
+
+        # Check for normality
+        normality_tests = {}
+        for column in numeric_columns:
+            _, p_value = stats.normaltest(data[column].dropna())
+            normality_tests[column] = {
+                'is_normal': p_value > 0.05,
+                'p_value': p_value
+            }
+        insights['normality_tests'] = normality_tests
+
+        return insights

text_analyzer.py

@@ -1,71 +0,0 @@
-import streamlit as st
-import pandas as pd
-import nltk
-from nltk.corpus import stopwords
-from nltk.tokenize import word_tokenize
-from nltk.sentiment import SentimentIntensityAnalyzer
-from collections import Counter
-from wordcloud import WordCloud
-import matplotlib.pyplot as plt
-
-nltk.download('punkt')
-nltk.download('stopwords')
-nltk.download('vader_lexicon')
-
-class TextAnalyzer:
-    def analyze_text(self, df):
-        text_columns = df.select_dtypes(include=['object']).columns
-        
-        if len(text_columns) > 0:
-            text_column = st.selectbox("Select text column for analysis", text_columns)
-            analysis_type = st.selectbox("Select analysis type", ["Word Frequency", "Sentiment Analysis", "Word Cloud"])
-            
-            if analysis_type == "Word Frequency":
-                self.perform_word_frequency(df[text_column])
-            elif analysis_type == "Sentiment Analysis":
-                self.perform_sentiment_analysis(df[text_column])
-            elif analysis_type == "Word Cloud":
-                self.generate_word_cloud(df[text_column])
-        else:
-            st.write("No text columns found in the dataset.")
-
-    def perform_word_frequency(self, text_series):
-        stop_words = set(stopwords.words('english'))
-        word_freq = Counter()
-        
-        for text in text_series:
-            tokens = word_tokenize(text.lower())
-            words = [word for word in tokens if word.isalnum() and word not in stop_words]
-            word_freq.update(words)
-        
-        st.subheader("Word Frequency Analysis")
-        n_words = st.slider("Select number of top words to display", min_value=5, max_value=50, value=20)
-        
-        top_words = word_freq.most_common(n_words)
-        fig = px.bar(x=[word for word, _ in top_words], y=[freq for _, freq in top_words], title="Top Words")
-        st.plotly_chart(fig)
-
-    def perform_sentiment_analysis(self, text_series):
-        sia = SentimentIntensityAnalyzer()
-        sentiments = text_series.apply(lambda x: sia.polarity_scores(x))
-        
-        st.subheader("Sentiment Analysis")
-        sentiment_df = pd.DataFrame(sentiments.tolist())
-        
-        fig = px.histogram(sentiment_df, x='compound', title="Sentiment Distribution")
-        st.plotly_chart(fig)
-        
-        st.write("Average Sentiment Scores:")
-        st.write(sentiment_df.mean())
-
-    def generate_word_cloud(self, text_series):
-        stop_words = set(stopwords.words('english'))
-        text = ' '.join(text_series)
-        
-        wordcloud = WordCloud(width=800, height=400, background_color='white', stopwords=stop_words).generate(text)
-        
-        st.subheader("Word Cloud")
-        fig, ax = plt.subplots()
-        ax.imshow(wordcloud, interpolation='bilinear')
-        ax.axis('off')
-        st.pyplot(fig)