Spaces:
Sleeping
Sleeping
File size: 4,922 Bytes
6acab55 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 | """
Loader model IndoBERT untuk SentiMart.
Model hasil fine-tuning (dari notebook, cell 6: `trainer.save_model(...)`)
diharapkan berada di folder `model/indobert_sentiment_final/` pada root
project ini. Struktur folder tersebut biasanya berisi:
config.json, model.safetensors (atau pytorch_model.bin),
tokenizer_config.json, vocab.txt, special_tokens_map.json
Cara mengisi folder ini:
1. Di notebook (Kaggle/Colab), setelah training selesai, download folder
'./indobert_sentiment_final/' (klik kanan -> Download, atau zip dulu).
2. Extract ke: sentimart/model/indobert_sentiment_final/
Jika model belum ada, app tetap bisa dijalankan dalam MODE DEMO (prediksi
dummy berbasis kata kunci) supaya wireframe & alur UI tetap bisa dicek.
"""
import os
import numpy as np
import streamlit as st
MODEL_DIR = os.path.join(os.path.dirname(os.path.dirname(__file__)), "model", "indobert_sentiment_final")
MAX_LENGTH = 128
LABEL_MAP = {0: "Negative", 1: "Positive"}
def model_is_available() -> bool:
return os.path.isdir(MODEL_DIR) and any(
f.startswith("config.json") for f in os.listdir(MODEL_DIR)
) if os.path.isdir(MODEL_DIR) else False
def get_model_debug_info() -> dict:
"""Info diagnostik untuk membantu cari tahu kenapa model tidak terdeteksi di deployment."""
model_root = os.path.dirname(MODEL_DIR)
info = {
"expected_model_dir": MODEL_DIR,
"model_root_exists": os.path.isdir(model_root),
"model_dir_exists": os.path.isdir(MODEL_DIR),
"files_in_model_root": [],
"files_in_model_dir": [],
"suspicious_small_files": [],
}
if os.path.isdir(model_root):
info["files_in_model_root"] = os.listdir(model_root)
if os.path.isdir(MODEL_DIR):
entries = os.listdir(MODEL_DIR)
info["files_in_model_dir"] = entries
for fname in entries:
fpath = os.path.join(MODEL_DIR, fname)
if os.path.isfile(fpath):
size = os.path.getsize(fpath)
# File model (.bin/.safetensors) yang cuma beberapa ratus byte biasanya
# adalah pointer Git LFS yang belum ter-resolve, bukan file asli.
if fname.endswith((".bin", ".safetensors")) and size < 10_000:
info["suspicious_small_files"].append((fname, size))
return info
@st.cache_resource(show_spinner=False)
def load_model():
"""Load tokenizer + model sekali saja, disimpan di cache Streamlit."""
import torch
from transformers import AutoTokenizer, AutoModelForSequenceClassification
tokenizer = AutoTokenizer.from_pretrained(MODEL_DIR)
model = AutoModelForSequenceClassification.from_pretrained(MODEL_DIR)
model.eval()
device = "cuda" if torch.cuda.is_available() else "cpu"
model.to(device)
return tokenizer, model, device
def _demo_predict(text: str):
"""Fallback berbasis kata kunci sederhana, dipakai kalau model belum di-copy."""
positive_words = ["bagus", "cepat", "puas", "recommended", "ramah", "sesuai", "mantap", "keren"]
negative_words = ["jelek", "rusak", "lambat", "kecewa", "buruk", "tidak sesuai", "lama", "cacat"]
t = text.lower()
pos_hits = sum(w in t for w in positive_words)
neg_hits = sum(w in t for w in negative_words)
if pos_hits == neg_hits:
label, conf = ("Positive", 0.55) if len(t) % 2 == 0 else ("Negative", 0.55)
elif pos_hits > neg_hits:
label, conf = "Positive", min(0.6 + 0.1 * pos_hits, 0.97)
else:
label, conf = "Negative", min(0.6 + 0.1 * neg_hits, 0.97)
probs = {label: conf, ("Negative" if label == "Positive" else "Positive"): 1 - conf}
return label, conf, probs
def predict_sentiment(text: str):
"""Prediksi satu review. Return: (label, confidence, {'Positive': p, 'Negative': p})"""
from .preprocessing import light_normalize
if not model_is_available():
return _demo_predict(text)
import torch
tokenizer, model, device = load_model()
clean_text = light_normalize(text)
inputs = tokenizer(
clean_text, truncation=True, padding="max_length",
max_length=MAX_LENGTH, return_tensors="pt",
).to(device)
with torch.no_grad():
logits = model(**inputs).logits
probs = torch.softmax(logits, dim=-1).cpu().numpy()[0]
pred_idx = int(np.argmax(probs))
label = LABEL_MAP[pred_idx]
confidence = float(probs[pred_idx])
prob_dict = {"Negative": float(probs[0]), "Positive": float(probs[1])}
return label, confidence, prob_dict
def predict_batch(texts: list[str], progress_callback=None):
"""Prediksi banyak review sekaligus. Return list of (label, confidence, prob_dict)."""
results = []
total = len(texts)
for i, t in enumerate(texts):
results.append(predict_sentiment(t))
if progress_callback is not None:
progress_callback((i + 1) / total)
return results
|