app.py

import os
import json
import re
import gradio as gr
import pandas as pd
import matplotlib
import matplotlib.pyplot as plt
import torch
import emoji

from apify_client import ApifyClient
from transformers import pipeline

matplotlib.use("Agg")

# Pastikan token Apify sudah tersimpan di Environment Variables/Secrets
APIFY_TOKEN = os.getenv("APIFY_FARIS")
ACTOR_ID = "nwua9Gu5YrADL7ZDj"

client = ApifyClient(APIFY_TOKEN) 

device = 0 if torch.cuda.is_available() else -1

MODEL_NAME = "Aardiiiiy/indobertweet-base-Indonesian-sentiment-analysis"

print("Memuat model sentimen...")
sentiment_model = pipeline(
    "text-classification",
    model=MODEL_NAME,
    tokenizer=MODEL_NAME,
    device=device
)
print("Model berhasil dimuat.")


def clean_review_text(text):
    """
    Fungsi pembersihan teks tingkat lanjut:
    1. Menghapus emoji secara presisi menggunakan library 'emoji'.
    2. Menghapus newline/tab yang merusak format tabel.
    3. Menghapus ulasan yang hanya berisi tanda baca.
    """
    if not text:
        return ""
    
    # 1. Hapus emoji secara absolut
    text = emoji.replace_emoji(text, replace='')
    
    # 2. Hapus enter, newline, dan tabulasi berlebih
    text = re.sub(r'[\r\n\t]+', ' ', text)
    
    # 3. Hapus spasi ganda
    text = re.sub(r'\s+', ' ', text)
    
    # Hapus spasi di awal/akhir kalimat
    text = text.strip()
    
    # 4. Validasi Akhir: Jika sisa teks HANYA berisi tanda baca (contoh sisa: ".", "?", "!!!", "-")
    if re.match(r'^[\W_]+$', text):
        return ""

    return text


def scrape_reviews(
    place_url,
    max_reviews,
    start_date
):

    run_input = {
        "startUrls": [{"url": place_url}],
        "maxCrawledPlacesPerSearch": 1,
        "maxReviews": int(max_reviews),
        "reviewsStartDate": start_date,
        "reviewsSort": "newest",
        "scrapeReviewsPersonalData": False,
        "scrapePlaceDetailPage": False,
        "scrapeTableReservationProvider": False,
        "scrapeOrderOnline": False,
        "includeWebResults": False,
        "scrapeDirectories": False,
        "scrapeContacts": False,
        "scrapeImageAuthors": False,
        "maximumLeadsEnrichmentRecords": 0,
        "skipClosedPlaces": True,
        "proxyConfig": {
            "useApifyProxy": True
        }
    }

    run = client.actor(
        ACTOR_ID
    ).call(
        run_input=run_input
    )

    dataset_id = run.default_dataset_id

    rows = []

    for item in client.dataset(
        dataset_id
    ).iterate_items():

        reviews = item.get(
            "reviews",
            []
        )

        for review in reviews:

            raw_text = review.get("text")
            date = review.get("publishedAtDate")

            if raw_text and date:
                
                # Gunakan fungsi cleaning yang sudah presisi
                cleaned_text = clean_review_text(raw_text)

                # Jika ulasan BUKAN string kosong setelah di-clean
                if cleaned_text: 

                    parsed_date = pd.to_datetime(date)

                    rows.append({
                        "bulan-tahun": parsed_date.strftime("%Y-%m"),
                        "ulasan": cleaned_text
                    })

    return rows


def analyze_sentiment(rows):

    reviews = [
        row["ulasan"]
        for row in rows
    ]

    predictions = sentiment_model(
        reviews,
        batch_size=8,
        truncation=True,
        max_length=512
    )

    sentiments = []
    
    # Mapping untuk mengubah label teks menjadi angka
    label_mapping = {
        "positive": 1,
        "neutral": 2,
        "negative": 3
    }

    for pred in predictions:
        # Konversi ke huruf kecil dulu (berjaga-jaga), lalu petakan ke angka
        raw_label = pred["label"].lower()
        numeric_label = label_mapping.get(raw_label, 2) # Default ke 2 jika ada error string
        sentiments.append(numeric_label)

    return sentiments


def create_visualization(
    sentiments,
    place_name
):
    
    # Menghitung berdasarkan label angka yang baru
    positive = sentiments.count(1)
    neutral = sentiments.count(2)
    negative = sentiments.count(3)

    fig = plt.figure(figsize=(6, 6))

    plt.pie(
        [positive, neutral, negative],
        labels=[
            "Positif",
            "Netral",
            "Negatif"
        ],
        autopct="%1.1f%%",
        startangle=140
    )

    plt.title(
        f"Overall Sentiment - {place_name}"
    )

    chart_path = "sentiment_chart.png"

    plt.savefig(
        chart_path,
        bbox_inches="tight"
    )

    plt.close()

    return chart_path


def export_csv(rows, sentiments):
    
    # Masukkan hasil sentimen angka ke dalam list rows
    for i in range(len(rows)):
        rows[i]["sentimen"] = sentiments[i]

    df = pd.DataFrame(rows)

    csv_path = "tourism_reviews.csv"

    df.to_csv(
        csv_path,
        index=False
    )

    return csv_path


def export_json(rows, sentiments):

    grouped = {}

    for i in range(len(rows)):

        month = rows[i]["bulan-tahun"]
        
        # Buat dictionary ulasan dan sentimen (angka)
        review_data = {
            "teks": rows[i]["ulasan"],
            "sentimen": sentiments[i]
        }

        if month not in grouped:
            grouped[month] = []

        grouped[month].append(review_data)

    json_output = []

    for month, reviews in grouped.items():

        json_output.append({
            "bulan-tahun": month,
            "total_ulasan": len(reviews),
            "ulasan": reviews
        })

    json_path = "monthly_summary.json"

    with open(
        json_path,
        "w",
        encoding="utf-8"
    ) as f:

        json.dump(
            json_output,
            f,
            ensure_ascii=False,
            indent=4
        )

    return json_path


def build_summary(
    sentiments,
    total_reviews
):
    
    # Menghitung berdasarkan label angka
    positive = sentiments.count(1)
    neutral = sentiments.count(2)
    negative = sentiments.count(3)

    return f"""
## Ringkasan Analisis

- Total Review (Setelah Cleaning): {total_reviews}

### Distribusi Sentimen

- Positif: {positive}
- Netral: {neutral}
- Negatif: {negative}
"""


def run_pipeline(
    place_name,
    place_url,
    max_reviews,
    start_month, # Input baru dari Dropdown Bulan
    start_year   # Input baru dari Dropdown Tahun
):
    
    # Kamus pemetaan bulan untuk merakit tanggal
    bulan_mapping = {
        "Januari": "01", "Februari": "02", "Maret": "03", "April": "04",
        "Mei": "05", "Juni": "06", "Juli": "07", "Agustus": "08",
        "September": "09", "Oktober": "10", "November": "11", "Desember": "12"
    }
    
    # Merakit tanggal (Format: YYYY-MM-DD), default tanggal 01
    bulan_angka = bulan_mapping.get(start_month, "01")
    start_date = f"{start_year}-{bulan_angka}-01"

    rows = scrape_reviews(
        place_url,
        max_reviews,
        start_date
    )

    if not rows:

        return (
            "Tidak ada data ulasan yang ditarik. (Semua ulasan di luar batas tanggal, atau hanya berisi emoji/kosong).",
            None,
            None,
            None
        )

    sentiments = analyze_sentiment(
        rows
    )

    chart_path = create_visualization(
        sentiments,
        place_name
    )

    csv_path = export_csv(
        rows,
        sentiments
    )

    json_path = export_json(
        rows,
        sentiments
    )

    summary = build_summary(
        sentiments,
        len(rows)
    )

    return (
        summary,
        chart_path,
        csv_path,
        json_path
    )


with gr.Blocks(
    title="Google Maps Tourism Review Scraper"
) as demo:

    gr.Markdown(
        "# Google Maps Tourism Review Scraper"
    )

    with gr.Row():

        place_name_input = gr.Textbox(
            label="Nama Destinasi (Hanya Untuk Label Chart)",
            placeholder="Malioboro"
        )
        
        place_url_input = gr.Textbox(
            label="Google Maps URL",
            placeholder="http://googleusercontent.com/maps.google.com/..."
        )

    with gr.Row():

        max_reviews_input = gr.Number(
            label="Max Reviews (Ketik Manual)",
            value=1000, 
            precision=0, 
            minimum=1
        )
        
        start_month_input = gr.Dropdown(
            choices=[
                "Januari", "Februari", "Maret", "April", 
                "Mei", "Juni", "Juli", "Agustus", 
                "September", "Oktober", "November", "Desember"
            ],
            value="Januari",
            label="Bulan Batas Mundur"
        )
        
        start_year_input = gr.Dropdown(
            # Menyesuaikan tahun agar mencakup 3 tahun ke belakang dari saat ini (2026)
            choices=["2023", "2024", "2025", "2026"],
            value="2023",
            label="Tahun Batas Mundur"
        )

    analyze_btn = gr.Button(
        "Scrape, Clean, & Analyze"
    )

    summary_output = gr.Markdown(
        label="Ringkasan"
    )

    result_chart = gr.Image(
        label="Overall Sentiment Visualization"
    )

    download_csv = gr.File(
        label="Download CSV Dataset"
    )

    download_json = gr.File(
        label="Download JSON Monthly Summary"
    )

    analyze_btn.click(
        run_pipeline,
        inputs=[
            place_name_input,
            place_url_input,
            max_reviews_input,
            start_month_input, # Mengganti Textbox dengan Dropdown Bulan
            start_year_input   # Mengganti Textbox dengan Dropdown Tahun
        ],
        outputs=[
            summary_output,
            result_chart,
            download_csv,
            download_json
        ]
    )

if __name__ == "__main__":
    demo.launch(ssr_mode=False)