Update app.py (#1)

- Update app.py (cc1cd60092728c60d255d67ed44fe0cb6f22b973)

app.py

@@ -1,14 +1,30 @@
 import gradio as gr
 import pandas as pd
 
+
+
+
+
+def nlp_pipeline(original_df):
+    original_df['Sum'] = df['a'] + df['b']
+    return original_df
+    
+
+
 def process_excel(file):
     try:
         # Ensure the file path is correct
         file_path = file.name if hasattr(file, 'name') else file
         # Read the Excel file
         df = pd.read_excel(file_path)
+        
         # Perform any processing on the DataFrame here
-        return df.head()  # Return the first few rows as an example
+        # Example: adding a new column with the sum of two other columns
+        # df['Sum'] = df['Column1'] + df['Column2']
+        result_df = nlp_pipeline(original_df)
+        
+        return result_df  # Return the first few rows as an example
+        
     except Exception as e:
         return str(e)  # Return the error message
 
@@ -24,3 +40,214 @@ interface = gr.Interface(
 # Launch the interface
 if __name__ == "__main__":
     interface.launch()
+
+
+
+
+
+
+
+
+
+
+
+# #!/usr/bin/env python
+# # coding: utf-8
+
+# import pandas as pd
+# import string
+# import nltk
+# import seaborn as sns
+# import matplotlib.pyplot as plt
+# from nltk.corpus import stopwords
+# from nltk.tokenize import word_tokenize
+# from nltk.sentiment import SentimentIntensityAnalyzer
+# from sklearn.feature_extraction.text import TfidfVectorizer
+# from sklearn.cluster import KMeans
+# from transformers import T5ForConditionalGeneration, T5Tokenizer
+# from datasets import Dataset
+
+# # Load the data
+# file_responses = pd.read_excel("#TaxDirection (Responses).xlsx")
+
+# # Process financial allocations
+# def process_allocations(df, col_name):
+#     return pd.to_numeric(df[col_name], errors='coerce').fillna(0)
+
+# columns_to_process = [
+#     '''Your financial allocation for Problem 1:
+# Mention the percentage of your Tax Amount which you wish the Government would allocate through their annual budget, to implement a specific solution for your 1st problem.''',
+#     '''Your financial allocation for Problem 2:
+# Mention the percentage of your Tax Amount which you wish the Government would allocate through their annual budget, to implement a solution specifically to your 2nd problem.''',
+#     '''Your financial allocation for Problem 3:
+# Mention the percentage of your Tax Amount which you wish the Government would allocate through their annual budget, to implement a solution specifically to your 3rd problem.'''
+# ]
+
+# for col in columns_to_process:
+#     file_responses[col] = process_allocations(file_responses, col)
+
+# file_responses['How much was your latest Tax payment (in U$D)?'] = pd.to_numeric(
+#     file_responses['How much was your latest Tax payment (in U$D)?'], errors='coerce').fillna(0)
+
+# # Compute total allocation and financial weights
+# file_responses['Total Allocation'] = file_responses[columns_to_process].apply(lambda x: x.clip(lower=10)).sum(axis=1)
+
+# for i in range(1, 4):
+#     file_responses[f'Financial Token Weight for Problem {i}'] = (
+#         file_responses['How much was your latest Tax payment (in U$D)?'] * 
+#         file_responses[columns_to_process[i - 1]] / 
+#         file_responses['Total Allocation']
+#     )
+
+# # Create initial datasets
+# initial_datasets = []
+# for i in range(1, 4):
+#     initial_datasets.append(
+#         file_responses[[f'''Describe Problem {i}:
+# Enter the context of the problem.
+# What are the difficulties you are facing personally or as a part of an organization?
+# You may briefly propose a solution idea as well.''',
+#             f'''Problem {i}: Geographical Location :
+# Where is the location you are facing this problem?
+# You may mention the nearby geographical area of the proposed solution as:
+# City/Town, State/Province, Country.''',
+#             f'Financial Token Weight for Problem {i}']]
+#     )
+
+# # Rename columns
+# for idx, df in enumerate(initial_datasets):
+#     initial_datasets[idx] = df.rename(columns={
+#         df.columns[0]: 'Problem_Description',
+#         df.columns[1]: 'Geographical_Location',
+#         df.columns[2]: 'Financial_Weight'
+#     })
+
+# # Merge datasets
+# merged_dataset = pd.concat(initial_datasets, ignore_index=True)
+
+# # Preprocess text
+# nltk.download('stopwords')
+# nltk.download('punkt')
+# nltk.download('omw-1.4')
+
+# def preprocess_text(text):
+#     translator = str.maketrans("", "", string.punctuation)
+#     text = text.translate(translator)
+#     tokens = word_tokenize(text)
+#     stop_words = set(stopwords.words('english'))
+#     tokens = [word for word in tokens if word.lower() not in stop_words]
+#     return ' '.join(tokens)
+
+# merged_dataset['Problem_Description'] = merged_dataset['Problem_Description'].astype(str).apply(preprocess_text)
+# merged_dataset['Problem_Description'] = merged_dataset['Problem_Description'].str.replace(r'\d+', '', regex=True)
+# merged_dataset['Geographical_Location'] = merged_dataset['Geographical_Location'].str.replace(r'\d+', '', regex=True)
+# merged_dataset['Problem_Description'] = merged_dataset['Problem_Description'].replace(r'http\S+', '', regex=True).replace(r'www\S+', '', regex=True)
+# merged_dataset['Geographical_Location'] = merged_dataset['Geographical_Location'].replace(r'http\S+', '', regex=True).replace(r'www\S+', '', regex=True)
+
+# # Lemmatize text
+# lemmatizer = nltk.WordNetLemmatizer()
+# merged_dataset['Problem_Description'] = merged_dataset['Problem_Description'].apply(lambda x: ' '.join([lemmatizer.lemmatize(word) for word in x.split()]))
+
+# # Clustering
+# corpus = merged_dataset['Problem_Description'].tolist()
+# tfidf_vectorizer = TfidfVectorizer(max_features=77000)
+# tfidf_matrix = tfidf_vectorizer.fit_transform(corpus)
+
+# problem_cluster_count = 77
+# kmeans = KMeans(n_clusters=problem_cluster_count)
+# kmeans.fit(tfidf_matrix)
+
+# terms = tfidf_vectorizer.get_feature_names_out()
+# ordered_centroids = kmeans.cluster_centers_.argsort()[:, ::-1]
+
+# cluster_representations = {}
+# for i in range(kmeans.n_clusters):
+#     cluster_representations[i] = [terms[ind] for ind in ordered_centroids[i, :17]]
+
+# merged_dataset['Problem_Category_Numeric'] = kmeans.labels_
+# merged_dataset['Problem_Category_Words'] = [cluster_representations[label] for label in kmeans.labels_]
+
+# # Clustering geographical locations
+# geographical_data = merged_dataset['Geographical_Location'].tolist()
+# tfidf_vectorizer_geography = TfidfVectorizer(max_features=3000)
+# tfidf_matrix_geography = tfidf_vectorizer_geography.fit_transform(geographical_data)
+
+# location_cluster_count = 33
+# kmeans_locations = KMeans(n_clusters=location_cluster_count)
+# kmeans_locations.fit(tfidf_matrix_geography)
+
+# terms_geography = tfidf_vectorizer_geography.get_feature_names_out()
+# ordered_centroids_geography = kmeans_locations.cluster_centers_.argsort()[:, ::-1]
+
+# cluster_representations_geography = {}
+# for i in range(kmeans_locations.n_clusters):
+#     cluster_representations_geography[i] = [terms_geography[ind] for ind in ordered_centroids_geography[i, :5]]
+
+# merged_dataset['Location_Category_Numeric'] = kmeans_locations.labels_
+# merged_dataset['Location_Category_Words'] = [cluster_representations_geography[label] for label in kmeans_locations.labels_]
+
+# # Create 2D matrices for problem descriptions and financial weights
+# matrix2Dfinances = [[[] for _ in range(location_cluster_count)] for _ in range(problem_cluster_count)]
+# matrix2Dproblems = [[[] for _ in range(location_cluster_count)] for _ in range(problem_cluster_count)]
+
+# for index, row in merged_dataset.iterrows():
+#     location_index = row['Location_Category_Numeric']
+#     problem_index = row['Problem_Category_Numeric']
+#     problem_description = row['Problem_Description']
+#     financial_wt = row['Financial_Weight']
+    
+#     matrix2Dproblems[problem_index][location_index].append(problem_description)
+#     matrix2Dfinances[problem_index][location_index].append(financial_wt)
+
+# # Aggregating financial weights
+# aggregated_Financial_wts = {}
+# un_aggregated_Financial_wts = {}
+
+# for Financ_wt_index, Financ_wt_row in enumerate(matrix2Dfinances):
+#     aggregated_Financial_wts[Financ_wt_index] = {}
+#     un_aggregated_Financial_wts[Financ_wt_index] = {}
+    
+#     for location_index, cell_finances in enumerate(Financ_wt_row):
+#         cell_sum = sum(cell_finances)
+#         aggregated_Financial_wts[Financ_wt_index][location_index] = cell_sum
+#         un_aggregated_Financial_wts[Financ_wt_index][location_index] = cell_finances
+
+# matrix2Dfinances_df = pd.DataFrame(aggregated_Financial_wts)
+# matrix2Dfinances_df.to_excel('matrix2Dfinances_HeatMap.xlsx', index=True)
+
+# unagregated_finances_df = pd.DataFrame(un_aggregated_Financial_wts)
+# unagregated_finances_df.to_excel('UNaggregated Financial Weights.xlsx', index=True)
+
+# # Create heatmaps
+# plt.figure(figsize=(15, 7))
+# sns.heatmap(matrix2Dfinances_df, annot=False, cmap='RdYlGn')
+# plt.title('Project Financial Weights')
+# plt.ylabel('Location Clusters')
+# plt.xlabel('Problem Clusters')
+# plt.savefig('Project Financial Weights_HeatMap_GreenHigh.png')
+# plt.show()
+
+# plt.figure(figsize=(14, 6))
+# sns.heatmap(matrix2Dfinances_df, annot=False, cmap='RdYlGn_r')
+# plt.title('Project Financial Weights')
+# plt.ylabel('Location Clusters')
+# plt.xlabel('Problem Clusters')
+# plt.savefig('Project Financial Weights_HeatMap_RedHigh.png')
+# plt.show()
+
+# # Summarizing problems using T5
+# model = T5ForConditionalGeneration.from_pretrained('t5-small')
+# tokenizer = T5Tokenizer.from_pretrained('t5-small')
+
+# def t5_summarize(text):
+#     input_text = "summarize: " + text
+#     inputs = tokenizer.encode(input_text, return_tensors="pt", max_length=512, truncation=True)
+#     summary_ids = model.generate(inputs, max_length=150, min_length=40, length_penalty=2.0, num_beams=4, early_stopping=True)
+#     return tokenizer.decode(summary_ids[0], skip_special_tokens=True)
+
+# summarized_problems = [[t5_summarize(" ".join(cell)) for cell in row] for row in matrix2Dproblems]
+
+# # Save summarized problems
+# with open('summarized_problems.txt', 'w') as file:
+#     for problem_row in summarized_problems:
+#         file.write("\t".join(problem_row) + "\n")