Spaces:
Sleeping
Sleeping
| import json | |
| import random | |
| import pickle | |
| import numpy as np | |
| import re | |
| import datetime | |
| import time # Import modul time untuk delay | |
| from flask import Flask, request, jsonify | |
| from sentence_transformers import SentenceTransformer | |
| from sklearn.metrics.pairwise import cosine_similarity | |
| class ImprovedBPJSChatbot: | |
| def __init__(self): | |
| self.load_models() | |
| self.load_intents() | |
| def load_models(self): | |
| """Load semua model yang diperlukan""" | |
| print("Memuat model dan konfigurasi...") | |
| # Load konfigurasi | |
| with open('model_config.pkl', 'rb') as f: | |
| config = pickle.load(f) | |
| # Load sentence transformer | |
| self.st_model = SentenceTransformer("Dyna-99/local-st-model") | |
| self.preprocessing_enabled = config['preprocessing_enabled'] | |
| # Load classifier | |
| with open('svm_model.pkl', 'rb') as f: | |
| self.clf = pickle.load(f) | |
| # Load label encoder | |
| with open('label_encoder.pkl', 'rb') as f: | |
| self.label_encoder = pickle.load(f) | |
| print("Semua model berhasil dimuat!") | |
| def load_intents(self): | |
| """Load data intents untuk responses""" | |
| with open('intents.json', 'r', encoding='utf-8') as f: | |
| self.intents_data = json.load(f) | |
| self.tag_responses = {intent['tag']: intent['responses'] for intent in self.intents_data['intents']} | |
| # Buat embeddings untuk semua patterns (untuk similarity fallback) | |
| self.pattern_embeddings = [] | |
| self.pattern_tags = [] | |
| for intent in self.intents_data['intents']: | |
| for pattern in intent['patterns']: | |
| processed_pattern = self.preprocess_text(pattern) if self.preprocessing_enabled else pattern | |
| embedding = self.st_model.encode(processed_pattern) | |
| self.pattern_embeddings.append(embedding) | |
| self.pattern_tags.append(intent['tag']) | |
| self.pattern_embeddings = np.array(self.pattern_embeddings) | |
| def preprocess_text(self, text): | |
| """Preprocessing teks yang sama dengan training""" | |
| text = text.lower() | |
| # Normalisasi singkatan | |
| text = re.sub(r'\bjkk\b', 'jaminan kecelakaan kerja', text) | |
| text = re.sub(r'\bjkm\b', 'jaminan kematian', text) | |
| text = re.sub(r'\bjht\b', 'jaminan hari tua', text) | |
| text = re.sub(r'\bjp\b', 'jaminan pensiun', text) | |
| text = re.sub(r'\bbpjs\b', 'bpjs ketenagakerjaan', text) | |
| # Hapus karakter khusus | |
| text = re.sub(r'[^\w\s]', ' ', text) | |
| text = re.sub(r'\s+', ' ', text).strip() | |
| return text | |
| def get_prediction_confidence(self, msg_embedding): | |
| """Dapatkan prediksi dengan confidence score""" | |
| # Prediksi probabilitas | |
| probabilities = self.clf.predict_proba(msg_embedding)[0] | |
| max_prob = np.max(probabilities) | |
| predicted_class = np.argmax(probabilities) | |
| predicted_tag = self.label_encoder.inverse_transform([predicted_class])[0] | |
| return predicted_tag, max_prob | |
| def similarity_fallback(self, msg_embedding, threshold=0.7): | |
| """Fallback menggunakan cosine similarity""" | |
| similarities = cosine_similarity(msg_embedding, self.pattern_embeddings)[0] | |
| max_similarity_idx = np.argmax(similarities) | |
| max_similarity = similarities[max_similarity_idx] | |
| if max_similarity >= threshold: | |
| return self.pattern_tags[max_similarity_idx], max_similarity | |
| return 'fallback', max_similarity | |
| def get_contextual_response(self, tag, user_message): | |
| """Pilih response yang paling kontekstual""" | |
| responses = self.tag_responses.get(tag, self.tag_responses['fallback']) | |
| # Jika hanya ada satu response, return langsung | |
| if len(responses) == 1: | |
| return responses[0] | |
| # Pilih response berdasarkan kata kunci dalam pesan user | |
| user_words = set(user_message.lower().split()) | |
| best_response = responses[0] | |
| best_score = 0 | |
| for response in responses: | |
| response_words = set(response.lower().split()) | |
| # Hitung kesamaan kata | |
| common_words = user_words.intersection(response_words) | |
| score = len(common_words) | |
| if score > best_score: | |
| best_score = score | |
| best_response = response | |
| # Jika tidak ada yang cocok, pilih random | |
| if best_score == 0: | |
| return random.choice(responses) | |
| return best_response | |
| def generate_response(self, message): | |
| """Generate response dengan multiple strategies""" | |
| if not message.strip(): | |
| return "Tolong kirim sebuah pesan." | |
| # Preprocessing | |
| processed_msg = self.preprocess_text(message) if self.preprocessing_enabled else message | |
| msg_embedding = self.st_model.encode(processed_msg).reshape(1, -1) | |
| # Strategy 1: SVM prediction dengan confidence | |
| predicted_tag, confidence = self.get_prediction_confidence(msg_embedding) | |
| # Strategy 2: Similarity fallback jika confidence rendah | |
| if confidence < 0.6: # Threshold bisa di-adjust | |
| fallback_tag, similarity = self.similarity_fallback(msg_embedding) | |
| if similarity > confidence: | |
| predicted_tag = fallback_tag | |
| # Strategy 3: Contextual response selection | |
| response = self.get_contextual_response(predicted_tag, message) | |
| # Logging untuk debugging | |
| print(f"Input: {message}") | |
| print(f"Processed: {processed_msg}") | |
| print(f"Predicted tag: {predicted_tag} (confidence: {confidence:.3f})") | |
| return response | |
| # Inisialisasi chatbot | |
| chatbot = ImprovedBPJSChatbot() | |
| # Flask app | |
| app = Flask(__name__) | |
| def chat(): | |
| try: | |
| msg = request.json.get("message", "").strip() | |
| time.sleep(1) | |
| response = chatbot.generate_response(msg) | |
| # Log untuk respons sukses | |
| print(f"[{datetime.datetime.now()}] 200 OK - Pesan masuk: '{msg}' -> Balasan: '{response}'") | |
| return jsonify({"reply": response}), 200 | |
| except Exception as e: | |
| # Log untuk error | |
| print(f"[{datetime.datetime.now()}] 500 ERROR - Pesan masuk: '{request.json}' -> Kesalahan: {e}") | |
| return jsonify({"reply": "Maaf, terjadi kesalahan sistem. Silakan coba lagi."}), 500 | |
| def health(): | |
| return jsonify({"status": "healthy", "model": "BPJS Chatbot Improved"}) | |
| if __name__ == '__main__': | |
| from waitress import serve | |
| serve(app, host='0.0.0.0', port=7860) | |
| # import json | |
| # import random | |
| # import pickle | |
| # import numpy as np | |
| # import re | |
| # from flask import Flask, request, jsonify | |
| # from sentence_transformers import SentenceTransformer | |
| # from sklearn.metrics.pairwise import cosine_similarity | |
| # # import os | |
| # # os.environ['HF_HOME'] = '/tmp/huggingface' | |
| # # os.environ['TRANSFORMERS_CACHE'] = '/tmp/huggingface/transformers' | |
| # # os.environ['HF_DATASETS_CACHE'] = '/tmp/huggingface/datasets' | |
| # # os.environ['HF_METRICS_CACHE'] = '/tmp/huggingface/metrics' | |
| # class ImprovedBPJSChatbot: | |
| # def __init__(self): | |
| # self.load_models() | |
| # self.load_intents() | |
| # def load_models(self): | |
| # """Load semua model yang diperlukan""" | |
| # print("Memuat model dan konfigurasi...") | |
| # # Load konfigurasi | |
| # with open('model_config.pkl', 'rb') as f: | |
| # config = pickle.load(f) | |
| # # Load sentence transformer | |
| # self.st_model = SentenceTransformer("Dyna-99/local-st-model") | |
| # self.preprocessing_enabled = config['preprocessing_enabled'] | |
| # # Load classifier | |
| # with open('svm_model.pkl', 'rb') as f: | |
| # self.clf = pickle.load(f) | |
| # # Load label encoder | |
| # with open('label_encoder.pkl', 'rb') as f: | |
| # self.label_encoder = pickle.load(f) | |
| # print("Semua model berhasil dimuat!") | |
| # def load_intents(self): | |
| # """Load data intents untuk responses""" | |
| # with open('intents.json', 'r', encoding='utf-8') as f: | |
| # self.intents_data = json.load(f) | |
| # self.tag_responses = {intent['tag']: intent['responses'] for intent in self.intents_data['intents']} | |
| # # Buat embeddings untuk semua patterns (untuk similarity fallback) | |
| # self.pattern_embeddings = [] | |
| # self.pattern_tags = [] | |
| # for intent in self.intents_data['intents']: | |
| # for pattern in intent['patterns']: | |
| # processed_pattern = self.preprocess_text(pattern) if self.preprocessing_enabled else pattern | |
| # embedding = self.st_model.encode(processed_pattern) | |
| # self.pattern_embeddings.append(embedding) | |
| # self.pattern_tags.append(intent['tag']) | |
| # self.pattern_embeddings = np.array(self.pattern_embeddings) | |
| # def preprocess_text(self, text): | |
| # """Preprocessing teks yang sama dengan training""" | |
| # text = text.lower() | |
| # # Normalisasi singkatan | |
| # text = re.sub(r'\bjkk\b', 'jaminan kecelakaan kerja', text) | |
| # text = re.sub(r'\bjkm\b', 'jaminan kematian', text) | |
| # text = re.sub(r'\bjht\b', 'jaminan hari tua', text) | |
| # text = re.sub(r'\bjp\b', 'jaminan pensiun', text) | |
| # text = re.sub(r'\bbpjs\b', 'bpjs ketenagakerjaan', text) | |
| # # Hapus karakter khusus | |
| # text = re.sub(r'[^\w\s]', ' ', text) | |
| # text = re.sub(r'\s+', ' ', text).strip() | |
| # return text | |
| # def get_prediction_confidence(self, msg_embedding): | |
| # """Dapatkan prediksi dengan confidence score""" | |
| # # Prediksi probabilitas | |
| # probabilities = self.clf.predict_proba(msg_embedding)[0] | |
| # max_prob = np.max(probabilities) | |
| # predicted_class = np.argmax(probabilities) | |
| # predicted_tag = self.label_encoder.inverse_transform([predicted_class])[0] | |
| # return predicted_tag, max_prob | |
| # def similarity_fallback(self, msg_embedding, threshold=0.7): | |
| # """Fallback menggunakan cosine similarity""" | |
| # similarities = cosine_similarity(msg_embedding, self.pattern_embeddings)[0] | |
| # max_similarity_idx = np.argmax(similarities) | |
| # max_similarity = similarities[max_similarity_idx] | |
| # if max_similarity >= threshold: | |
| # return self.pattern_tags[max_similarity_idx], max_similarity | |
| # return 'fallback', max_similarity | |
| # def get_contextual_response(self, tag, user_message): | |
| # """Pilih response yang paling kontekstual""" | |
| # responses = self.tag_responses.get(tag, self.tag_responses['fallback']) | |
| # # Jika hanya ada satu response, return langsung | |
| # if len(responses) == 1: | |
| # return responses[0] | |
| # # Pilih response berdasarkan kata kunci dalam pesan user | |
| # user_words = set(user_message.lower().split()) | |
| # best_response = responses[0] | |
| # best_score = 0 | |
| # for response in responses: | |
| # response_words = set(response.lower().split()) | |
| # # Hitung kesamaan kata | |
| # common_words = user_words.intersection(response_words) | |
| # score = len(common_words) | |
| # if score > best_score: | |
| # best_score = score | |
| # best_response = response | |
| # # Jika tidak ada yang cocok, pilih random | |
| # if best_score == 0: | |
| # return random.choice(responses) | |
| # return best_response | |
| # def generate_response(self, message): | |
| # """Generate response dengan multiple strategies""" | |
| # if not message.strip(): | |
| # return "Tolong kirim sebuah pesan." | |
| # # Preprocessing | |
| # processed_msg = self.preprocess_text(message) if self.preprocessing_enabled else message | |
| # msg_embedding = self.st_model.encode(processed_msg).reshape(1, -1) | |
| # # Strategy 1: SVM prediction dengan confidence | |
| # predicted_tag, confidence = self.get_prediction_confidence(msg_embedding) | |
| # # Strategy 2: Similarity fallback jika confidence rendah | |
| # if confidence < 0.6: # Threshold bisa di-adjust | |
| # fallback_tag, similarity = self.similarity_fallback(msg_embedding) | |
| # if similarity > confidence: | |
| # predicted_tag = fallback_tag | |
| # # Strategy 3: Contextual response selection | |
| # response = self.get_contextual_response(predicted_tag, message) | |
| # # Logging untuk debugging | |
| # print(f"Input: {message}") | |
| # print(f"Processed: {processed_msg}") | |
| # print(f"Predicted tag: {predicted_tag} (confidence: {confidence:.3f})") | |
| # return response | |
| # # Inisialisasi chatbot | |
| # chatbot = ImprovedBPJSChatbot() | |
| # # Flask app | |
| # app = Flask(__name__) | |
| # @app.route('/chat', methods=['POST']) | |
| # def chat(): | |
| # try: | |
| # msg = request.json.get("message", "").strip() | |
| # response = chatbot.generate_response(msg) | |
| # return jsonify({"reply": response}) | |
| # except Exception as e: | |
| # print(f"Error: {e}") | |
| # return jsonify({"reply": "Maaf, terjadi kesalahan sistem. Silakan coba lagi."}) | |
| # @app.route('/health', methods=['GET']) | |
| # def health(): | |
| # return jsonify({"status": "healthy", "model": "BPJS Chatbot Improved"}) | |
| # if __name__ == '__main__': | |
| # from waitress import serve | |
| # serve(app, host='0.0.0.0', port=7860) | |