import matplotlib.pyplot as plt import plotly.express as px import pandas as pd import gradio as gr from google_play_scraper import Sort, reviews, app from datetime import datetime, timedelta import io import google.generativeai as genai import re from nltk.corpus import stopwords from Sastrawi.Stemmer.StemmerFactory import StemmerFactory from Sastrawi.StopWordRemover.StopWordRemoverFactory import StopWordRemoverFactory import pickle import nltk nltk.download('stopwords') # Tujuan file disimpan destination_file_1y_ex3 = 'data/app_reviews_1y_ex3.csv' model_file = 'model/best_model.pkl' vectorizer_file = 'model/vectorizer.pkl' # Global variables to store API key and model name api_key = None model_name = "gemini-2.0-flash" # Default model name with open(model_file, 'rb') as file: best_model = pickle.load(file) with open(vectorizer_file, 'rb') as file: vectorizer = pickle.load(file) # Cache stop words indonesian_stopwords = stopwords.words('indonesian') # Create stemmer factory = StemmerFactory() stemmer = factory.create_stemmer() # Create stop word remover stopword_factory = StopWordRemoverFactory() stopword_remover = stopword_factory.create_stop_word_remover() def preprocess_text(text): # 1. Handle None values if text is None: return "" # Or any other suitable replacement # Lowercase and remove punctuation & special characters in one step text = re.sub(r'[^\w\s\d]+', '', text.lower()) # Remove extra whitespaces text = re.sub(r'\s+', ' ', text).strip() # Stemming and stop word removal using NLTK and list comprehension text = stemmer.stem(text) # Indonesian stemming text = stopword_remover.remove(text) # Indonesian stopword removal words = text.split() words = [word for word in words if word not in indonesian_stopwords] # Remove Indonesian stopwords text = " ".join(words) return text def predict_sentiment(text): # Preprocess the input text processed_text = preprocess_text(text) # Transform the text using the loaded vectorizer text_vectorized = vectorizer.transform([processed_text]) # Predict the sentiment prediction = best_model.predict(text_vectorized)[0] return prediction # Fungsi untuk melakukan labeling dengan gemini api def label_sentiment_with_gemini(text, api_key, model_name): """Melakukan labeling sentimen menggunakan Gemini.""" prompt = f"""Klasifikasikan sentimen ulasan berikut menjadi: '1.puas', '2.tidak puas', '3.netral'. Perhatikan sarkasme dan sindiran, atau ekspresi negatif/positif halus, serta bahasa yang digunakan. **Ulasan:** {text} **Tampilkan hanya Sentimen** """ try: genai.configure(api_key=api_key) # Konfigurasi Gemini API di dalam fungsi model = genai.GenerativeModel(model_name) response = model.generate_content(prompt) generated_content = response.text.strip().lower() generated_content = re.sub(' ', '', generated_content) if "1.puas" in generated_content: return "puas" elif "2.tidakpuas" in generated_content: return "tidak puas" else: return "netral" except genai.errors.ResourceExhaustedError: print("Error: Rate limit exceeded. Please try again later.") return "netral" # or another appropriate default value except Exception as e: print(f"An unexpected error occurred: {e}") return "netral" # or another appropriate default value def predict_and_label(text): try: if not text: # Check if the text input is empty raise gr.Error("Please enter a review.") # Raise a Gradio error with a message if not api_key: # Check if the api_key input is empty raise gr.Error("Please enter your correct API_KEY on API Settings.") # Raise a Gradio error with a message if not model_file: # Check if the model_name input is empty raise gr.Error("Please enter your correct MODEL_NAME on API Settings.") # Raise a Gradio error with a message prediction = predict_sentiment(text) # Konversi np.str_ menjadi str prediction = prediction.item() # Atau prediction.astype(str) label_gemini = label_sentiment_with_gemini(text, api_key, model_name) return prediction, label_gemini except (ValueError, TypeError, AttributeError) as e: # Catch specific errors related to data types, empty inputs, and unexpected values raise gr.Error(f"Error processing input: {type(e).__name__}. Please check your input.") except genai.errors.ResourceExhaustedError: # Handle rate limit exceeded error raise gr.Error("Error: Rate limit exceeded for Gemini API/You forgot to update API_KEY. Please try again later.") except Exception as e: # Catch any other unexpected errors raise gr.Error(f"An unexpected error occurred: {type(e).__name__}. Please try again later.") def update_api_credentials(new_api_key, new_model_name): global api_key, model_name # Access the global variables api_key = str(new_api_key) model_name = str(new_model_name) #test api and show successfull if connected try: genai.configure(api_key=api_key) # Konfigurasi Gemini API di dalam fungsi model = genai.GenerativeModel(model_name) response = model.generate_content("Test API Connection.Just say Yes if successfull") generated_content = response.text.strip().lower() except genai.errors.ResourceExhaustedError: print("Error: Rate limit exceeded. Please try again later.") return "Error: Rate limit exceeded. Please try again later." except Exception as e: print(f"An unexpected error occurred: {e}") return "An unexpected error occurred. Please try again later." print(f"API Key: {api_key}") print(f"Model Name: {model_name}") return generated_content, " API credentials updated successfully!" def scrape_and_show_data(): try: # List App Packages app_packages = [ 'id.dana', #Dana 'com.shopeepay.id', #Shopeepay 'com.gojek.gopay', #Gopay 'ovo.id', #Ovo ] language = 'id' country = 'id' app_reviews = [] current_date = datetime.now() one_year_ago = current_date - timedelta(days=365) for ap in app_packages: for score in [1, 2, 3, 4, 5]: # Ambil semua rating (1-5) rvs, _ = reviews( ap, lang=language, country=country, sort=Sort.NEWEST, # Hanya ambil ulasan terbaru (newest) count=10, # Sesuaikan jumlah ulasan yang ingin di-scrape filter_score_with=score ) # Filter ulasan untuk satu tahun terakhir for r in rvs: review_date = datetime.strptime(r['at'].strftime("%Y-%m-%d"), "%Y-%m-%d") if review_date >= one_year_ago: r['sortOrder'] = 'newest' # Tetapkan sortOrder menjadi 'newest' r['appId'] = ap app_reviews.append(r) df = pd.DataFrame(app_reviews) # Buat label (misal: score 4-5 puas, 3 netral, 1-2 nggak puas) def label_sentiment(score): if score >= 4: return 'puas' elif score < 3: return 'tidak_puas' else: return 'netral' df['rating'] = df['score'].apply(label_sentiment) # Load Apps Info app_infos = [] for ap in app_packages: info = app(ap, lang=language, country=country) del info['comments'] app_infos.append(info) app_infos_df = pd.DataFrame(app_infos) df = pd.merge(df, app_infos_df[['appId', 'title']], on='appId', how='left') df = df.sort_values(by='at', ascending=False).head(10) # predict the data with predict_and_label. The result have 2 list, example is ('puas', 'netral'). Put to dataframe for column predict_model and predict_gemini df['predict_model'], df['predict_gemini'] = zip(*df['content'].apply(predict_and_label)) # show only column at rename as date, content, rating, and order desc by date df = df[['title','at', 'content', 'score', 'rating','predict_model','predict_gemini']].rename(columns={'at': 'date'}) return df except Exception as e: raise gr.Error(f"Error scraping data: {type(e).__name__}. Please check your app package names and connection.") def scrape_and_download_data(app_packages, language, country, sort, score, start_date, end_date, count): try: app_reviews = [] # Convert app_packages to a list if it's a string if isinstance(app_packages, str): app_packages = [app_packages] # Convert date strings to datetime objects (if needed) if isinstance(start_date, str): start_date = datetime.strptime(start_date, "%Y-%m-%d") if isinstance(end_date, str): end_date = datetime.strptime(end_date, "%Y-%m-%d") # Scrape data based on criteria for ap in app_packages: for scr in str(score): # Ambil semua rating (1-5) rvs, _ = reviews( ap, lang=str(language), # Convert language to string country=str(country), # Convert country to string sort=Sort.NEWEST if str(sort) == 'NEWEST' else Sort.MOST_RELEVANT, count=int(count), filter_score_with=scr, ) # Filter reviews based on date range and other criteria for r in rvs: review_date = datetime.strptime(r['at'].strftime("%Y-%m-%d"), "%Y-%m-%d") if start_date <= review_date <= end_date: # Date range filter r['sortOrder'] = sort r['appId'] = ap app_reviews.append(r) # Create DataFrame df = pd.DataFrame(app_reviews) # Check if DataFrame is empty if df.empty: # Handle empty DataFrame, e.g., return an empty DataFrame or raise an exception print("DataFrame is empty. No reviews found for the specified criteria.") return df # or: raise ValueError("No reviews found for the specified criteria.") else: # Load Apps Info app_infos = [] for ap in app_packages: info = app(ap, lang=language, country=country) del info['comments'] app_infos.append(info) app_infos_df = pd.DataFrame(app_infos) df = pd.merge(df, app_infos_df[['appId', 'title']], on='appId', how='left') # Create label if DataFrame is not empty def label_sentiment(score): if score >= 4: return 'puas' elif score < 3: return 'tidak_puas' else: return 'netral' df['rating'] = df['score'].apply(label_sentiment) # show only column title, at, sortOrder, reviewId, userName, userImage, content, score, thumbsUpCount, replyContent, repliedAt, rating df = df[['title','at', 'sortOrder', 'reviewId', 'userName', 'userImage', 'content', 'score', 'thumbsUpCount', 'replyContent', 'repliedAt', 'rating']].rename(columns={'at': 'date'}) # Rename 'at' to 'date df = df.sort_values(by='date', ascending=False) return df except Exception as e: raise gr.Error(f"Error scraping or processing data: {type(e).__name__}. Please check your inputs and connection.") # def create_charts(): # # 1. Rating Distribution Pie Chart # df = scrape_and_show_data() # rating_counts = df['rating'].value_counts() # # Create the pie chart using Matplotlib # fig_pie, ax_pie = plt.subplots() # ax_pie.pie(rating_counts, labels=rating_counts.index, autopct='%1.1f%%', startangle=90) # ax_pie.axis('equal') # Equal aspect ratio ensures that pie is drawn as a circle. # plt.title("Rating Distribution") # # Convert to gradio plot # rating_pie_chart = gr.Plot(value=fig_pie) # Using gr.Plot # # 2. Daily Reviews Line Chart # daily_reviews = df.groupby('date').size().reset_index(name='total_reviews') # # Create line chart using Plotly # fig_line = px.line(daily_reviews, x='date', y='total_reviews', title='Total Reviews per Day') # fig_line.update_traces(mode='markers+lines') # # Convert to gradio plot # daily_reviews_chart = gr.Plot(value=fig_line) # Using gr.Plot # return rating_pie_chart, daily_reviews_chart # Return both gradio plots with gr.Blocks() as apps: with gr.Tabs(): with gr.TabItem("Prediction Existing Data"): # Sentiment Prediction section gr.Interface( fn=predict_and_label, inputs=[ gr.Textbox(lines=5, label="Masukkan Ulasan"), ], outputs=[ gr.Textbox(label="Prediksi Model",info="Prediksi Model Sentiment"), gr.Textbox(label="Prediksi Gemini",info="Prediksi Gemini Sentiment"), ], title="Prediksi Sentimen dari Ulasan Aplikasi di Google Play Store", description="Masukkan ulasan Anda untuk memprediksi sentimen.", api_name="prediksi_sentimen" ) gr.Interface( fn=scrape_and_show_data, inputs=None, outputs=gr.Dataframe(label="Cleaned Reviews DataFrame",wrap=True), description="Displaying the Latest the Data:", api_name="prediksi_sentimen_latest" ) # gr.Interface( # fn=create_charts, # inputs=None, # outputs=[ # gr.Plot(label="Rating Distribution"), # gr.Plot(label="Daily Reviews"), # ], # description="Displaying Charts:", # ) with gr.TabItem("Download New Data"): with gr.Column(): # Place input elements in a column app_packages_input = gr.Textbox(label="App Packages (comma-separated)", value="com.gojek.gopay",info="Enter app packages separated by commas") language_input = gr.Textbox(label="Language", value="id", info="Enter language code") country_input = gr.Textbox(label="Country", value="id", info="Enter country code") sort_input = gr.Radio(["NEWEST", "MOST_RELEVANT"], label="Sort Order", value="NEWEST", info="Select sort order") scores_input = gr.CheckboxGroup([1, 2, 3, 4, 5], label="Scores", value=[1, 2, 3, 4, 5], info="Select scores") start_date_input = gr.Textbox(label="Start Date (YYYY-MM-DD)", value=(datetime.now() - timedelta(days=365)).strftime("%Y-%m-%d"),info="Enter start date (YYYY-MM-DD)") end_date_input = gr.Textbox(label="End Date (YYYY-MM-DD)", value=datetime.now().strftime("%Y-%m-%d"),info="Enter end date (YYYY-MM-DD)") count = gr.Textbox(label="Count", value="10",info="Enter count") generate_button = gr.Button("Generate Data") # download_button = gr.DownloadButton(label="Download Data") # Place output elements below the input column output_data = gr.Dataframe(label="Scraped Data", wrap=True) generate_button.click( fn=scrape_and_download_data, inputs=[app_packages_input, language_input, country_input, sort_input, scores_input, start_date_input, end_date_input, count], outputs=[output_data], api_name="generate_data" ) # download_button.click( # fn=lambda df: io.StringIO(df.to_csv(index=False)), # Convert DataFrame to CSV in memory # inputs=output_data, # outputs=download_button, # api_name="download_data" # ) with gr.TabItem("API Settings"): # New tab for API settings with gr.Row(): api_key_input = gr.Textbox(label="API Key", value="", info="Enter your API key") model_name_input = gr.Textbox(label="Model Name", value="gemini-2.0-flash", info="Enter the model name") update_button = gr.Button("Check and Update API Credentials") update_button.click( fn=update_api_credentials, inputs=[api_key_input, model_name_input], outputs=gr.Textbox(label="Status"), api_name="update_api_credentials" ) # information to get API Key on https://aistudio.google.com/app/apikey gr.Markdown("Get API Key on https://aistudio.google.com/app/apikey") apps.launch(share=False, debug=True, auth=("admin", "admin"), ssr_mode=False)