"
+ ],
+ "text/plain": [
+ "GridSearchCV(cv=5,\n",
+ " estimator=Pipeline(steps=[('indobert', IndoBertEncoder()),\n",
+ " ('clf',\n",
+ " LogisticRegression(max_iter=1000,\n",
+ " n_jobs=-1))]),\n",
+ " n_jobs=-1,\n",
+ " param_grid={'clf__C': [0.1, 1.0, 5.0],\n",
+ " 'clf__class_weight': [None, 'balanced']},\n",
+ " verbose=2)"
+ ]
+ },
+ "execution_count": 31,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "grid_logreg_indobert.fit(X_train, y_train)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 32,
+ "id": "dab3db37",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Best params (LogReg): {'clf__C': 5.0, 'clf__class_weight': 'balanced'}\n",
+ "Best CV score (LogReg): 0.7617846971710651\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(\"Best params (LogReg):\", grid_logreg_indobert.best_params_)\n",
+ "print(\"Best CV score (LogReg):\", grid_logreg_indobert.best_score_)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 33,
+ "id": "88c544aa",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "best_logreg_indobert = grid_logreg_indobert.best_estimator_"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 34,
+ "id": "0f7ad22c",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Test Accuracy (LogReg IndoBERT): 0.767\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "y_pred_logreg_indobert = best_logreg_indobert.predict(X_test)\n",
+ "acc_logreg_indobert = accuracy_score(y_test, y_pred_logreg_indobert)\n",
+ "print(f\"Test Accuracy (LogReg IndoBERT): {acc_logreg_indobert:.3f}\\n\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 35,
+ "id": "bfd79866",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Classification Report (LogReg IndoBERT):\n",
+ " precision recall f1-score support\n",
+ "\n",
+ " akses_repository 0.77 0.77 0.77 13\n",
+ "cari_buku_isbn_callnumber 0.92 0.85 0.88 13\n",
+ " cari_buku_judul 0.56 0.77 0.65 13\n",
+ " cari_buku_penulis 0.67 0.77 0.71 13\n",
+ " cari_buku_topik 0.75 0.46 0.57 13\n",
+ " cari_rekomendasi 0.77 0.77 0.77 13\n",
+ " cek_ketersediaan_buku 0.70 0.54 0.61 13\n",
+ " donasi_buku 0.80 0.92 0.86 13\n",
+ " info_denda 1.00 0.85 0.92 13\n",
+ " jam_buka 0.77 0.77 0.77 13\n",
+ " lainnya 0.62 0.77 0.69 13\n",
+ " layanan_ejournal_ebook 0.62 0.62 0.62 13\n",
+ " layanan_ruang_diskusi 0.86 0.92 0.89 13\n",
+ " layanan_turnitin 0.92 0.92 0.92 13\n",
+ " lokasi_buku_rak 0.67 0.77 0.71 13\n",
+ " lokasi_perpustakaan 1.00 0.92 0.96 13\n",
+ " panduan_peminjaman 0.62 0.62 0.62 13\n",
+ " panduan_pengembalian 0.67 0.92 0.77 13\n",
+ " panduan_perpanjangan 0.89 0.62 0.73 13\n",
+ " salam 0.77 0.71 0.74 14\n",
+ " tanya_fungsi_mlibbot 1.00 0.92 0.96 13\n",
+ " tata_tertib 0.82 0.69 0.75 13\n",
+ "\n",
+ " accuracy 0.77 287\n",
+ " macro avg 0.78 0.77 0.77 287\n",
+ " weighted avg 0.78 0.77 0.77 287\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(\"Classification Report (LogReg IndoBERT):\")\n",
+ "print(classification_report(y_test, y_pred_logreg_indobert))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 36,
+ "id": "3adb28b8",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "Analisis Intent Paling Sulit (LogReg IndoBERT):\n",
+ " Intent F1-Score Precision Recall TP FP FN\n",
+ " cari_buku_topik 0.5714 0.7500 0.4615 6 2 7\n",
+ " cek_ketersediaan_buku 0.6087 0.7000 0.5385 7 3 6\n",
+ "layanan_ejournal_ebook 0.6154 0.6154 0.6154 8 5 5\n",
+ " panduan_peminjaman 0.6154 0.6154 0.6154 8 5 5\n",
+ " cari_buku_judul 0.6452 0.5556 0.7692 10 8 3\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(\"\\nAnalisis Intent Paling Sulit (LogReg IndoBERT):\")\n",
+ "df_indobert = analyze_intent_difficulty(y_test, y_pred_logreg_indobert, best_logreg_indobert)\n",
+ "print(df_indobert[[\"Intent\", \"F1-Score\", \"Precision\", \"Recall\", \"TP\", \"FP\", \"FN\"]].head(5).to_string(index=False))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 37,
+ "id": "896ade18",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "pipe_nb = Pipeline([\n",
+ " (\"tfidf\", TfidfVectorizer(\n",
+ " preprocessor=None,\n",
+ " lowercase=False\n",
+ " )),\n",
+ " (\"clf\", MultinomialNB())\n",
+ "])"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 38,
+ "id": "8f7361d0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "param_grid_nb = {\n",
+ " \"tfidf__ngram_range\": [(1, 1), (1, 2)],\n",
+ " \"tfidf__min_df\": [1, 2],\n",
+ " \"clf__alpha\": [0.1, 0.5, 1.0]\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 39,
+ "id": "4d4eb089",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "grid_nb = GridSearchCV(\n",
+ " pipe_nb,\n",
+ " param_grid_nb,\n",
+ " cv=5,\n",
+ " n_jobs=-1,\n",
+ " verbose=2\n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 40,
+ "id": "a9e9ad6b",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Fitting 5 folds for each of 12 candidates, totalling 60 fits\n"
+ ]
+ },
+ {
+ "data": {
+ "text/html": [
+ "
GridSearchCV(cv=5,\n",
+ " estimator=Pipeline(steps=[('tfidf',\n",
+ " TfidfVectorizer(lowercase=False)),\n",
+ " ('clf', MultinomialNB())]),\n",
+ " n_jobs=-1,\n",
+ " param_grid={'clf__alpha': [0.1, 0.5, 1.0], 'tfidf__min_df': [1, 2],\n",
+ " 'tfidf__ngram_range': [(1, 1), (1, 2)]},\n",
+ " verbose=2) In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. \n",
+ "
\n",
+ "
\n",
+ " Parameters \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " estimator\n",
+ " estimator: estimator object \n",
+ " \n",
+ " \n",
+ " Pipeline(step...inomialNB())]) \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " param_grid\n",
+ " param_grid: dict or list of dictionaries \n",
+ " \n",
+ " \n",
+ " {'clf__alpha': [0.1, 0.5, ...], 'tfidf__min_df': [1, 2], 'tfidf__ngram_range': [(1, ...), (1, ...)]} \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " scoring\n",
+ " scoring: str, callable, list, tuple or dict, default=None` is used. \n",
+ " \n",
+ " \n",
+ " None \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " n_jobs\n",
+ " n_jobs: int, default=None` \n",
+ " \n",
+ " \n",
+ " -1 \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " refit\n",
+ " refit: bool, str, or callable, default=True` \n",
+ " \n",
+ " \n",
+ " True \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " cv\n",
+ " cv: int, cross-validation generator or an iterable, default=None` for the various \n",
+ " \n",
+ " \n",
+ " 5 \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " verbose\n",
+ " verbose: int \n",
+ " \n",
+ " \n",
+ " 2 \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " pre_dispatch\n",
+ " pre_dispatch: int, or str, default='2*n_jobs' \n",
+ " \n",
+ " \n",
+ " '2*n_jobs' \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " error_score\n",
+ " error_score: 'raise' or numeric, default=np.nan \n",
+ " \n",
+ " \n",
+ " nan \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " return_train_score\n",
+ " return_train_score: bool, default=False \n",
+ " \n",
+ " \n",
+ " False \n",
+ " \n",
+ " \n",
+ " \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n",
+ " Parameters \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " input\n",
+ " input: {'filename', 'file', 'content'}, default='content' \n",
+ " \n",
+ " \n",
+ " 'content' \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " encoding\n",
+ " encoding: str, default='utf-8' \n",
+ " \n",
+ " \n",
+ " 'utf-8' \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " decode_error\n",
+ " decode_error: {'strict', 'ignore', 'replace'}, default='strict' \n",
+ " \n",
+ " \n",
+ " 'strict' \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " strip_accents\n",
+ " strip_accents: {'ascii', 'unicode'} or callable, default=None \n",
+ " \n",
+ " \n",
+ " None \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " lowercase\n",
+ " lowercase: bool, default=True \n",
+ " \n",
+ " \n",
+ " False \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " preprocessor\n",
+ " preprocessor: callable, default=None \n",
+ " \n",
+ " \n",
+ " None \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " tokenizer\n",
+ " tokenizer: callable, default=None \n",
+ " \n",
+ " \n",
+ " None \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " analyzer\n",
+ " analyzer: {'word', 'char', 'char_wb'} or callable, default='word' \n",
+ " \n",
+ " \n",
+ " 'word' \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " stop_words\n",
+ " stop_words: {'english'}, list, default=None \n",
+ " \n",
+ " \n",
+ " None \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " token_pattern\n",
+ " token_pattern: str, default=r\"(?u)\\\\b\\\\w\\\\w+\\\\b\" \n",
+ " \n",
+ " \n",
+ " '(?u)\\\\b\\\\w\\\\w+\\\\b' \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " ngram_range\n",
+ " ngram_range: tuple (min_n, max_n), default=(1, 1) \n",
+ " \n",
+ " \n",
+ " (1, ...) \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " max_df\n",
+ " max_df: float or int, default=1.0 \n",
+ " \n",
+ " \n",
+ " 1.0 \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " min_df\n",
+ " min_df: float or int, default=1 \n",
+ " \n",
+ " \n",
+ " 1 \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " max_features\n",
+ " max_features: int, default=None \n",
+ " \n",
+ " \n",
+ " None \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " vocabulary\n",
+ " vocabulary: Mapping or iterable, default=None \n",
+ " \n",
+ " \n",
+ " None \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " binary\n",
+ " binary: bool, default=False \n",
+ " \n",
+ " \n",
+ " False \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " dtype\n",
+ " dtype: dtype, default=float64 \n",
+ " \n",
+ " \n",
+ " <class 'numpy.float64'> \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " norm\n",
+ " norm: {'l1', 'l2'} or None, default='l2' \n",
+ " \n",
+ " \n",
+ " 'l2' \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " use_idf\n",
+ " use_idf: bool, default=True \n",
+ " \n",
+ " \n",
+ " True \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " smooth_idf\n",
+ " smooth_idf: bool, default=True \n",
+ " \n",
+ " \n",
+ " True \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " sublinear_tf\n",
+ " sublinear_tf: bool, default=False \n",
+ " \n",
+ " \n",
+ " False \n",
+ " \n",
+ " \n",
+ " \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
\n",
+ "
\n",
+ "
\n",
+ " Parameters \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " \n",
+ " alpha\n",
+ " alpha: float or array-like of shape (n_features,), default=1.0 \n",
+ " \n",
+ " \n",
+ " 0.1 \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " force_alpha\n",
+ " force_alpha: bool, default=True \n",
+ " \n",
+ " \n",
+ " True \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " fit_prior\n",
+ " fit_prior: bool, default=True \n",
+ " \n",
+ " \n",
+ " True \n",
+ " \n",
+ " \n",
+ "\n",
+ " \n",
+ " \n",
+ " \n",
+ " class_prior\n",
+ " class_prior: array-like of shape (n_classes,), default=None \n",
+ " \n",
+ " \n",
+ " None \n",
+ " \n",
+ " \n",
+ " \n",
+ "
\n",
+ " \n",
+ "
\n",
+ "
"
+ ],
+ "text/plain": [
+ "GridSearchCV(cv=5,\n",
+ " estimator=Pipeline(steps=[('tfidf',\n",
+ " TfidfVectorizer(lowercase=False)),\n",
+ " ('clf', MultinomialNB())]),\n",
+ " n_jobs=-1,\n",
+ " param_grid={'clf__alpha': [0.1, 0.5, 1.0], 'tfidf__min_df': [1, 2],\n",
+ " 'tfidf__ngram_range': [(1, 1), (1, 2)]},\n",
+ " verbose=2)"
+ ]
+ },
+ "execution_count": 40,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "grid_nb.fit(X_train, y_train)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 41,
+ "id": "7b7acd19",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Best params (NB): {'clf__alpha': 0.1, 'tfidf__min_df': 1, 'tfidf__ngram_range': (1, 2)}\n",
+ "Best CV score (NB): 0.7731573951015759\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(\"Best params (NB):\", grid_nb.best_params_)\n",
+ "print(\"Best CV score (NB):\", grid_nb.best_score_)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 42,
+ "id": "66191eef",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "best_nb = grid_nb.best_estimator_"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 43,
+ "id": "50a22d39",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Test Accuracy (Naive Bayes): 0.787\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "y_pred_nb = best_nb.predict(X_test)\n",
+ "acc_nb = accuracy_score(y_test, y_pred_nb)\n",
+ "print(f\"Test Accuracy (Naive Bayes): {acc_nb:.3f}\\n\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 44,
+ "id": "5bf52ba7",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Classification Report (Naive Bayes):\n",
+ " precision recall f1-score support\n",
+ "\n",
+ " akses_repository 0.73 0.85 0.79 13\n",
+ "cari_buku_isbn_callnumber 0.86 0.92 0.89 13\n",
+ " cari_buku_judul 0.86 0.92 0.89 13\n",
+ " cari_buku_penulis 0.73 0.62 0.67 13\n",
+ " cari_buku_topik 0.71 0.77 0.74 13\n",
+ " cari_rekomendasi 0.85 0.85 0.85 13\n",
+ " cek_ketersediaan_buku 0.90 0.69 0.78 13\n",
+ " donasi_buku 0.57 0.62 0.59 13\n",
+ " info_denda 0.78 0.54 0.64 13\n",
+ " jam_buka 0.92 0.92 0.92 13\n",
+ " lainnya 0.75 0.92 0.83 13\n",
+ " layanan_ejournal_ebook 0.77 0.77 0.77 13\n",
+ " layanan_ruang_diskusi 0.92 0.92 0.92 13\n",
+ " layanan_turnitin 0.67 0.77 0.71 13\n",
+ " lokasi_buku_rak 0.71 0.77 0.74 13\n",
+ " lokasi_perpustakaan 0.92 0.92 0.92 13\n",
+ " panduan_peminjaman 0.71 0.77 0.74 13\n",
+ " panduan_pengembalian 0.58 0.54 0.56 13\n",
+ " panduan_perpanjangan 0.77 0.77 0.77 13\n",
+ " salam 1.00 0.86 0.92 14\n",
+ " tanya_fungsi_mlibbot 0.87 1.00 0.93 13\n",
+ " tata_tertib 0.80 0.62 0.70 13\n",
+ "\n",
+ " accuracy 0.79 287\n",
+ " macro avg 0.79 0.79 0.78 287\n",
+ " weighted avg 0.79 0.79 0.79 287\n",
+ "\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(\"Classification Report (Naive Bayes):\")\n",
+ "print(classification_report(y_test, y_pred_nb))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 45,
+ "id": "5825a038",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "Analisis Intent Paling Sulit (Naive Bayes):\n",
+ " Intent F1-Score Precision Recall TP FP FN\n",
+ "panduan_pengembalian 0.5600 0.5833 0.5385 7 5 6\n",
+ " donasi_buku 0.5926 0.5714 0.6154 8 6 5\n",
+ " info_denda 0.6364 0.7778 0.5385 7 2 6\n",
+ " cari_buku_penulis 0.6667 0.7273 0.6154 8 3 5\n",
+ " tata_tertib 0.6957 0.8000 0.6154 8 2 5\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(\"\\nAnalisis Intent Paling Sulit (Naive Bayes):\")\n",
+ "df_nb = analyze_intent_difficulty(y_test, y_pred_nb, best_nb)\n",
+ "print(df_nb[[\"Intent\", \"F1-Score\", \"Precision\", \"Recall\", \"TP\", \"FP\", \"FN\"]].head(5).to_string(index=False))"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "086da60e",
+ "metadata": {
+ "lines_to_next_cell": 2
+ },
+ "source": [
+ "Pada percobaan ini, model Naive Bayes tetap menggunakan fitur TF-IDF dan tidak digabung dengan embedding IndoBERT. Alasannya karena secara prinsip, Multinomial Naive Bayes dirancang untuk bekerja dengan fitur berupa frekuensi kata atau bobot yang mirip frekuensi (seperti count dan TF-IDF) yang bernilai non-negatif.\n",
+ "\n",
+ "Sementara itu, embedding IndoBERT berbentuk vektor dens dengan nilai kontinu yang bisa positif maupun negatif, dan tidak lagi merepresentasikan \"jumlah kemunculan kata\", tetapi makna kalimat di ruang vektor. Tipe fitur seperti ini tidak sesuai dengan asumsi probabilistik Multinomial Naive Bayes, sehingga performanya justru bisa tidak stabil atau menurun."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 46,
+ "id": "7324de39",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "LogReg (TF-IDF) Test Accuracy : 0.826\n",
+ "Naive Bayes (TF-IDF) Test Accuracy : 0.787\n",
+ "LogReg (IndoBERT) Test Accuracy : 0.767\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(f\"LogReg (TF-IDF) Test Accuracy : {acc_logreg:.3f}\")\n",
+ "print(f\"Naive Bayes (TF-IDF) Test Accuracy : {acc_nb:.3f}\")\n",
+ "print(f\"LogReg (IndoBERT) Test Accuracy : {acc_logreg_indobert:.3f}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 47,
+ "id": "c7f802ea",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "candidates = {\n",
+ " \"logreg_tfidf\": (acc_logreg, best_logreg),\n",
+ " \"naive_bayes_tfidf\": (acc_nb, best_nb),\n",
+ " \"logreg_indobert\": (acc_logreg_indobert, best_logreg_indobert),\n",
+ "}"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 48,
+ "id": "10f797ff",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "best_model_name, (best_acc, final_model) = max(\n",
+ " candidates.items(),\n",
+ " key=lambda item: item[1][0] \n",
+ ")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 49,
+ "id": "950677e7",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Chosen model: logreg_tfidf (accuracy = 0.826)\n"
+ ]
+ }
+ ],
+ "source": [
+ "print(f\"Chosen model: {best_model_name} (accuracy = {best_acc:.3f})\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 50,
+ "id": "67fbe771",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "\n",
+ "Menyimpan semua model...\n"
+ ]
+ }
+ ],
+ "source": [
+ "# Simpan semua model\n",
+ "print(\"\\nMenyimpan semua model...\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 51,
+ "id": "e2f4f6a0",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Saved LogReg TF-IDF to: model/intent_model_logreg_tfidf.pkl\n"
+ ]
+ }
+ ],
+ "source": [
+ "logreg_tfidf_path = \"model/intent_model_logreg_tfidf.pkl\"\n",
+ "joblib.dump(best_logreg, logreg_tfidf_path)\n",
+ "print(f\"Saved LogReg TF-IDF to: {logreg_tfidf_path}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 52,
+ "id": "2095be1c",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Saved Naive Bayes TF-IDF to: model/intent_model_naive_bayes_tfidf.pkl\n"
+ ]
+ }
+ ],
+ "source": [
+ "nb_tfidf_path = \"model/intent_model_naive_bayes_tfidf.pkl\"\n",
+ "joblib.dump(best_nb, nb_tfidf_path)\n",
+ "print(f\"Saved Naive Bayes TF-IDF to: {nb_tfidf_path}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 53,
+ "id": "f228591c",
+ "metadata": {
+ "lines_to_next_cell": 1
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "Saved LogReg IndoBERT to: model/intent_model_logreg_indobert.pkl\n"
+ ]
+ }
+ ],
+ "source": [
+ "indobert_path = \"model/intent_model_logreg_indobert.pkl\"\n",
+ "joblib.dump(best_logreg_indobert, indobert_path)\n",
+ "print(f\"Saved LogReg IndoBERT to: {indobert_path}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 54,
+ "id": "8fef94f8",
+ "metadata": {
+ "lines_to_next_cell": 1
+ },
+ "outputs": [],
+ "source": [
+ "def predict_intent_sentence(s):\n",
+ " s_clean = preprocess(s)\n",
+ " return final_model.predict([s_clean])[0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 55,
+ "id": "e930cca5",
+ "metadata": {
+ "lines_to_next_cell": 2
+ },
+ "outputs": [],
+ "source": [
+ "tests = [\n",
+ " \"jam buka perpustakaan hari sabtu\",\n",
+ " \"perpus maranatha buka sampe jam berapa ya?\",\n",
+ " \"besok minggu perpus buka gak?\",\n",
+ " \"jam operasional perpustakaan pas libur nasional gimana?\",\n",
+ " \"hari ini perpus udah buka belum?\",\n",
+ "\n",
+ " \"ada buku basis data fathansyah gak\",\n",
+ " \"ada buku tentang machine learning terbaru gak?\",\n",
+ " \"cek dong buku pemrograman python masih tersedia ga\",\n",
+ " \"di perpus ada novel laskar pelangi gak sih?\",\n",
+ " \"kalo mau cari skripsi tentang data mining ada ga?\",\n",
+ "\n",
+ " \"cara booking ruang diskusi gimana\",\n",
+ " \"book ruang belajar kelompok bisa lewat mana?\",\n",
+ " \"ruang diskusi bisa dipake berapa jam maksimal?\",\n",
+ " \"bisa reservasi ruang belajar lewat online gak?\",\n",
+ "\n",
+ " \"kalau telat balikin buku dendanya berapa\",\n",
+ " \"telat ngembaliin buku 2 hari berapa ya?\",\n",
+ " \"Kalau saya telat mengembalikan, konsekuensinya apa?\",\n",
+ " \"kalau hilangin buku perpus dendanya gimana ya?\",\n",
+ " \"batas maksimal telat pengembalian sebelum kena blokir berapa hari?\",\n",
+ "\n",
+ " \"cara akses e journal dari luar kampus\",\n",
+ " \"akses database journal lewat wifi kos bisa gak?\",\n",
+ " \"punya akses ke ieee atau sciencedirect gak ya?\",\n",
+ " \"login e-resources pake akun apa ya?\",\n",
+ " \"kalo lupa password e journal harus gimana?\",\n",
+ "\n",
+ " \"perpus maranatha ada dmn sih\",\n",
+ " \"alamat lengkap perpustakaan maranatha di mana ya?\",\n",
+ " \"nomor telepon perpustakaan ada?\",\n",
+ " \"perpus ada di gedung mana ya di kampus?\",\n",
+ "\n",
+ " \"cara pinjam buku di perpus gimana\",\n",
+ " \"bisa perpanjang peminjaman buku lewat online gak?\",\n",
+ " \"kalo mau pinjem buku harus bawa ktm gak?\",\n",
+ " \"maksimal bisa pinjam berapa buku sekaligus?\",\n",
+ " \"lama peminjaman buku berapa hari ya?\",\n",
+ "\n",
+ " \"halo mlibbot\",\n",
+ " \"hi bot, bisa bantu cari buku?\",\n",
+ " \"p\",\n",
+ " \"halo, ini perpus maranatha ya?\",\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 56,
+ "id": "cbda5478",
+ "metadata": {
+ "lines_to_next_cell": 1
+ },
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "'jam buka perpustakaan hari sabtu' -> jam_buka\n",
+ "'perpus maranatha buka sampe jam berapa ya?' -> jam_buka\n",
+ "'besok minggu perpus buka gak?' -> jam_buka\n",
+ "'jam operasional perpustakaan pas libur nasional gimana?' -> jam_buka\n",
+ "'hari ini perpus udah buka belum?' -> jam_buka\n",
+ "'ada buku basis data fathansyah gak' -> cari_buku_judul\n",
+ "'ada buku tentang machine learning terbaru gak?' -> cari_rekomendasi\n",
+ "'cek dong buku pemrograman python masih tersedia ga' -> cek_ketersediaan_buku\n",
+ "'di perpus ada novel laskar pelangi gak sih?' -> cari_buku_judul\n",
+ "'kalo mau cari skripsi tentang data mining ada ga?' -> cari_buku_topik\n",
+ "'cara booking ruang diskusi gimana' -> layanan_ruang_diskusi\n",
+ "'book ruang belajar kelompok bisa lewat mana?' -> layanan_ruang_diskusi\n",
+ "'ruang diskusi bisa dipake berapa jam maksimal?' -> layanan_ruang_diskusi\n",
+ "'bisa reservasi ruang belajar lewat online gak?' -> panduan_perpanjangan\n",
+ "'kalau telat balikin buku dendanya berapa' -> info_denda\n",
+ "'telat ngembaliin buku 2 hari berapa ya?' -> info_denda\n",
+ "'Kalau saya telat mengembalikan, konsekuensinya apa?' -> panduan_pengembalian\n",
+ "'kalau hilangin buku perpus dendanya gimana ya?' -> panduan_pengembalian\n",
+ "'batas maksimal telat pengembalian sebelum kena blokir berapa hari?' -> info_denda\n",
+ "'cara akses e journal dari luar kampus' -> layanan_ejournal_ebook\n",
+ "'akses database journal lewat wifi kos bisa gak?' -> layanan_ejournal_ebook\n",
+ "'punya akses ke ieee atau sciencedirect gak ya?' -> layanan_ejournal_ebook\n",
+ "'login e-resources pake akun apa ya?' -> cari_buku_isbn_callnumber\n",
+ "'kalo lupa password e journal harus gimana?' -> layanan_ejournal_ebook\n",
+ "'perpus maranatha ada dmn sih' -> lokasi_perpustakaan\n",
+ "'alamat lengkap perpustakaan maranatha di mana ya?' -> lokasi_perpustakaan\n",
+ "'nomor telepon perpustakaan ada?' -> lokasi_perpustakaan\n",
+ "'perpus ada di gedung mana ya di kampus?' -> lokasi_perpustakaan\n",
+ "'cara pinjam buku di perpus gimana' -> panduan_peminjaman\n",
+ "'bisa perpanjang peminjaman buku lewat online gak?' -> panduan_perpanjangan\n",
+ "'kalo mau pinjem buku harus bawa ktm gak?' -> panduan_peminjaman\n",
+ "'maksimal bisa pinjam berapa buku sekaligus?' -> panduan_peminjaman\n",
+ "'lama peminjaman buku berapa hari ya?' -> panduan_peminjaman\n",
+ "'halo mlibbot' -> salam\n",
+ "'hi bot, bisa bantu cari buku?' -> tanya_fungsi_mlibbot\n",
+ "'p' -> lainnya\n",
+ "'halo, ini perpus maranatha ya?' -> salam\n"
+ ]
+ }
+ ],
+ "source": [
+ "for t in tests:\n",
+ " print(f\"{t!r} -> {predict_intent_sentence(t)}\")"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "id": "df1eac42",
+ "metadata": {},
+ "source": [
+ "Test dengan IndoBERT model"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 57,
+ "id": "5678285a",
+ "metadata": {
+ "lines_to_next_cell": 1
+ },
+ "outputs": [],
+ "source": [
+ "def predict_intent_sentence(s):\n",
+ " s_clean = preprocess(s)\n",
+ " return best_logreg_indobert.predict([s_clean])[0]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 58,
+ "id": "373f65c0",
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "tests = [\n",
+ " \"jam buka perpustakaan hari sabtu\",\n",
+ " \"perpus maranatha buka sampe jam berapa ya?\",\n",
+ " \"besok minggu perpus buka gak?\",\n",
+ " \"jam operasional perpustakaan pas libur nasional gimana?\",\n",
+ " \"hari ini perpus udah buka belum?\",\n",
+ "\n",
+ " \"ada buku basis data fathansyah gak\",\n",
+ " \"ada buku tentang machine learning terbaru gak?\",\n",
+ " \"cek dong buku pemrograman python masih tersedia ga\",\n",
+ " \"di perpus ada novel laskar pelangi gak sih?\",\n",
+ " \"kalo mau cari skripsi tentang data mining ada ga?\",\n",
+ "\n",
+ " \"cara booking ruang diskusi gimana\",\n",
+ " \"book ruang belajar kelompok bisa lewat mana?\",\n",
+ " \"ruang diskusi bisa dipake berapa jam maksimal?\",\n",
+ " \"bisa reservasi ruang belajar lewat online gak?\",\n",
+ "\n",
+ " \"kalau telat balikin buku dendanya berapa\",\n",
+ " \"telat ngembaliin buku 2 hari berapa ya?\",\n",
+ " \"Kalau saya telat mengembalikan, konsekuensinya apa?\",\n",
+ " \"kalau hilangin buku perpus dendanya gimana ya?\",\n",
+ " \"batas maksimal telat pengembalian sebelum kena blokir berapa hari?\",\n",
+ "\n",
+ " \"cara akses e journal dari luar kampus\",\n",
+ " \"akses database journal lewat wifi kos bisa gak?\",\n",
+ " \"punya akses ke ieee atau sciencedirect gak ya?\",\n",
+ " \"login e-resources pake akun apa ya?\",\n",
+ " \"kalo lupa password e journal harus gimana?\",\n",
+ "\n",
+ " \"perpus maranatha ada dmn sih\",\n",
+ " \"alamat lengkap perpustakaan maranatha di mana ya?\",\n",
+ " \"nomor telepon perpustakaan ada?\",\n",
+ " \"perpus ada di gedung mana ya di kampus?\",\n",
+ "\n",
+ " \"cara pinjam buku di perpus gimana\",\n",
+ " \"bisa perpanjang peminjaman buku lewat online gak?\",\n",
+ " \"kalo mau pinjem buku harus bawa ktm gak?\",\n",
+ " \"maksimal bisa pinjam berapa buku sekaligus?\",\n",
+ " \"lama peminjaman buku berapa hari ya?\",\n",
+ "\n",
+ " \"halo mlibbot\",\n",
+ " \"hi bot, bisa bantu cari buku?\",\n",
+ " \"p\",\n",
+ " \"halo, ini perpus maranatha ya?\",\n",
+ "]"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 59,
+ "id": "85d65c9f",
+ "metadata": {},
+ "outputs": [
+ {
+ "name": "stdout",
+ "output_type": "stream",
+ "text": [
+ "'jam buka perpustakaan hari sabtu' -> jam_buka\n",
+ "'perpus maranatha buka sampe jam berapa ya?' -> jam_buka\n",
+ "'besok minggu perpus buka gak?' -> salam\n",
+ "'jam operasional perpustakaan pas libur nasional gimana?' -> jam_buka\n",
+ "'hari ini perpus udah buka belum?' -> salam\n",
+ "'ada buku basis data fathansyah gak' -> cari_buku_penulis\n",
+ "'ada buku tentang machine learning terbaru gak?' -> cari_buku_judul\n",
+ "'cek dong buku pemrograman python masih tersedia ga' -> cari_buku_judul\n",
+ "'di perpus ada novel laskar pelangi gak sih?' -> cari_buku_penulis\n",
+ "'kalo mau cari skripsi tentang data mining ada ga?' -> cari_buku_judul\n",
+ "'cara booking ruang diskusi gimana' -> layanan_ruang_diskusi\n",
+ "'book ruang belajar kelompok bisa lewat mana?' -> layanan_ruang_diskusi\n",
+ "'ruang diskusi bisa dipake berapa jam maksimal?' -> layanan_ruang_diskusi\n",
+ "'bisa reservasi ruang belajar lewat online gak?' -> layanan_ejournal_ebook\n",
+ "'kalau telat balikin buku dendanya berapa' -> info_denda\n",
+ "'telat ngembaliin buku 2 hari berapa ya?' -> panduan_perpanjangan\n",
+ "'Kalau saya telat mengembalikan, konsekuensinya apa?' -> info_denda\n",
+ "'kalau hilangin buku perpus dendanya gimana ya?' -> info_denda\n",
+ "'batas maksimal telat pengembalian sebelum kena blokir berapa hari?' -> panduan_perpanjangan\n",
+ "'cara akses e journal dari luar kampus' -> layanan_ejournal_ebook\n",
+ "'akses database journal lewat wifi kos bisa gak?' -> layanan_ejournal_ebook\n",
+ "'punya akses ke ieee atau sciencedirect gak ya?' -> layanan_ejournal_ebook\n",
+ "'login e-resources pake akun apa ya?' -> cari_buku_judul\n",
+ "'kalo lupa password e journal harus gimana?' -> layanan_ejournal_ebook\n",
+ "'perpus maranatha ada dmn sih' -> lokasi_perpustakaan\n",
+ "'alamat lengkap perpustakaan maranatha di mana ya?' -> lokasi_perpustakaan\n",
+ "'nomor telepon perpustakaan ada?' -> jam_buka\n",
+ "'perpus ada di gedung mana ya di kampus?' -> lokasi_perpustakaan\n",
+ "'cara pinjam buku di perpus gimana' -> panduan_peminjaman\n",
+ "'bisa perpanjang peminjaman buku lewat online gak?' -> panduan_perpanjangan\n",
+ "'kalo mau pinjem buku harus bawa ktm gak?' -> panduan_peminjaman\n",
+ "'maksimal bisa pinjam berapa buku sekaligus?' -> panduan_peminjaman\n",
+ "'lama peminjaman buku berapa hari ya?' -> panduan_perpanjangan\n",
+ "'halo mlibbot' -> salam\n",
+ "'hi bot, bisa bantu cari buku?' -> tanya_fungsi_mlibbot\n",
+ "'p' -> cari_buku_penulis\n",
+ "'halo, ini perpus maranatha ya?' -> salam\n"
+ ]
+ }
+ ],
+ "source": [
+ "for t in tests:\n",
+ " print(f\"{t!r} -> {predict_intent_sentence(t)}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "id": "46d71456-0a12-472d-ad1d-7b37cd481a21",
+ "metadata": {},
+ "outputs": [],
+ "source": []
+ }
+ ],
+ "metadata": {
+ "jupytext": {
+ "cell_metadata_filter": "-all",
+ "main_language": "python",
+ "notebook_metadata_filter": "-all"
+ },
+ "kernelspec": {
+ "display_name": "Python 3 (ipykernel)",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.11.14"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 5
+}