"""ACOS Course Evaluation System · Streamlit App Supports English and Bahasa Melayu input. BM input → translated to English → ACOS inference → 4-column bilingual result table. """ import os, io, re, time, random import streamlit as st import pandas as pd st.set_page_config( page_title="ACOS · Course Evaluation", page_icon="🎓", layout="wide", initial_sidebar_state="expanded", ) # ── HuggingFace Hub model ID (only change from local version) ───────────────── HF_MODEL_ID = "lijinzheyy/ACOS_CourseEvaluation_System" HISTORY = { "Epoch": [1, 2, 3, 4, 5, 6, 7, 8], "Train Loss": [5.4623, 1.8059, 1.1240, 0.8544, 0.7015, 0.6106, 0.5562, 0.5288], "Val F1": [0.0607, 0.1190, 0.1507, 0.1866, 0.2094, 0.2054, 0.2282, 0.2308], "Precision": [0.0758, 0.1248, 0.1604, 0.1983, 0.2234, 0.2155, 0.2373, 0.2420], "Recall": [0.0506, 0.1137, 0.1422, 0.1761, 0.1969, 0.1963, 0.2198, 0.2205], } SAMPLES_EN = [ ("Instructor clarity", "The instructor explains concepts very clearly and responds to questions quickly. Best course I've taken so far."), ("Mixed feedback", "The instructor explains concepts clearly, but the assignments are far too vague and the grading is inconsistent."), ("Pacing issue", "Excellent course content! The topics are relevant, but the pacing is way too fast for beginners."), ("Poor support", "Forum support is very slow and the lecture notes are completely disorganised. Very disappointed."), ("Positive overall", "The hands-on projects are practical and well-designed. Highly recommended for anyone in data science."), ("Grading unclear", "Great instructor who responds quickly to emails. The grading criteria is unclear and confusing though."), ("Content heavy", "The course content is too heavy and the weekly readings take forever. The quizzes are fair though."), ("Bad teaching", "Bad teaching, random, disorganized. Hardly any explanation of concepts. You'll learn almost nothing."), ("Easy and useful", "Useful and easy to follow. A bit dry in terms of delivery, but the material itself is solid."), ("Highly negative", "He was very rude and did not respect his students. I would not take any more of his courses."), ] SAMPLES_MS = [ ("Pensyarah bagus", "Pensyarah menjelaskan konsep dengan sangat jelas dan mudah difahami. Kursus terbaik yang pernah saya ikuti sejauh ini."), ("Tugasan tidak jelas", "Pensyarah baik dan peramah, tetapi tugasan terlalu kabur dan markah tidak konsisten. Agak mengecewakan."), ("Kandungan terlalu berat", "Kandungan kursus terlalu berat dan bacaan mingguan mengambil masa yang sangat lama. Namun begitu, kuiz adalah adil."), ("Sokongan forum lemah", "Sokongan forum sangat lambat dan nota kuliah tidak tersusun langsung. Saya sangat kecewa dengan kursus ini."), ("Projek praktikal", "Projek hands-on sangat praktikal dan direka dengan baik. Saya sangat mengesyorkan kursus ini untuk sesiapa dalam bidang sains data."), ("Pengajaran buruk", "Pengajaran sangat buruk, tidak tersusun dan tiada penjelasan konsep yang mencukupi. Anda hampir tidak akan belajar apa-apa."), ("Kadar pembelajaran laju", "Kandungan kursus sangat relevan tetapi kadar pembelajaran terlalu laju untuk pelajar baru dalam bidang ini."), ("Maklum balas campuran", "Pensyarah sangat baik dan membalas emel dengan cepat, tetapi kriteria penilaian tidak jelas dan mengelirukan."), ] PLACEHOLDER_EN = "— Select English sample —" PLACEHOLDER_MS = "— Pilih sampel Bahasa Melayu —" MARIAN_MS_EN = "Helsinki-NLP/opus-mt-mul-en" MS_NORMALISE = [ (r'\be-mel\b', 'emel'), (r'\be-learning\b', 'elearning'), (r'\be-book\b', 'ebook'), (r'\bhand-on\b', 'hands on'), ] BM_DICT = { "positive": "positif", "negative": "negatif", "neutral": "neutral", "implicit": "tersirat", "course": "kursus", "course general": "kursus am", "course quality": "kualiti kursus", "course content": "kandungan kursus", "instructor": "pensyarah", "lecturer": "pensyarah", "teacher": "guru", "assignment": "tugasan", "assignments": "tugasan", "grading": "penilaian", "grade": "gred", "grades": "gred", "grading criteria": "kriteria penilaian", "content": "kandungan", "material": "bahan", "materials": "bahan", "pacing": "kadar pembelajaran", "pace": "kadar", "support": "sokongan", "forum": "forum", "forum support": "sokongan forum", "quiz": "kuiz", "quizzes": "kuiz", "project": "projek", "projects": "projek", "lecture notes": "nota kuliah", "notes": "nota", "reading": "bacaan", "readings": "bacaan", "teaching": "pengajaran", "clear": "jelas", "very clear": "sangat jelas", "clearly": "dengan jelas", "good": "baik", "very good": "sangat baik", "excellent": "cemerlang", "great": "hebat", "practical": "praktikal", "well-designed": "direka dengan baik", "useful": "berguna", "easy": "mudah", "easy to follow": "mudah diikuti", "solid": "mantap", "relevant": "relevan", "very relevant": "sangat relevan", "fair": "adil", "recommended": "disyorkan", "quick": "cepat", "quickly": "dengan cepat", "friendly": "mesra", "helpful": "membantu", "organised": "tersusun", "organized": "tersusun", "well organised": "tersusun dengan baik", "slow": "lambat", "very slow": "sangat lambat", "fast": "laju", "too fast": "terlalu laju", "heavy": "berat", "too heavy": "terlalu berat", "vague": "kabur", "too vague": "terlalu kabur", "unclear": "tidak jelas", "confusing": "mengelirukan", "inconsistent": "tidak konsisten", "disorganised": "tidak tersusun", "disorganized": "tidak tersusun", "disappointed": "kecewa", "very disappointed": "sangat kecewa", "disappointing": "mengecewakan", "bad": "buruk", "poor": "lemah", "rude": "kasar", "random": "tidak tentu hala", "short": "pendek", "too short": "terlalu pendek", "long": "lama", "forever": "terlalu lama", } def to_bm(text: str) -> str: key = text.lower().strip() return BM_DICT.get(key, text) def sanitize_filename(name: str) -> str: name = name.encode("ascii", "ignore").decode("ascii") name = re.sub(r"[^\w]", "_", name) name = re.sub(r"_+", "_", name).strip("_") return name or "batch" def normalise_ms(text: str) -> str: for pattern, replacement in MS_NORMALISE: text = re.sub(pattern, replacement, text, flags=re.IGNORECASE) return text def is_valid_translation(result: str, original: str) -> bool: if not result or not result.strip(): return False alpha_chars = sum(1 for c in result if c.isalpha()) alpha_ratio = alpha_chars / max(len(result), 1) if alpha_ratio < 0.4: return False if result.strip().lower() == original.strip().lower(): return False return True # ── CSS ────────────────────────────────────────────────────────────────────── st.markdown(""" """, unsafe_allow_html=True) # ── Session state ───────────────────────────────────────────────────────────── defaults = { "model": None, "tokenizer": None, "model_loaded": False, "ms_en_tok": None, "ms_en_mdl": None, "translators_loaded": False, "trans_error": "", "review_text": "", "batch_results": [], "upload_filename": "batch", "ta_key": 0, "last_sample_label": "", "last_loaded_en": PLACEHOLDER_EN, "last_loaded_ms": PLACEHOLDER_MS, "en_sel_ver": 0, "ms_sel_ver": 0, } for k, v in defaults.items(): if k not in st.session_state: st.session_state[k] = v # ── Loaders ─────────────────────────────────────────────────────────────────── @st.cache_resource(show_spinner=False) def load_model(): from transformers import T5ForConditionalGeneration, T5Tokenizer tok = T5Tokenizer.from_pretrained(HF_MODEL_ID) mdl = T5ForConditionalGeneration.from_pretrained(HF_MODEL_ID) mdl.eval() return tok, mdl @st.cache_resource(show_spinner=False) def load_ms_en_model(): from transformers import MarianMTModel, MarianTokenizer tok = MarianTokenizer.from_pretrained(MARIAN_MS_EN) mdl = MarianMTModel.from_pretrained(MARIAN_MS_EN) mdl.eval() return tok, mdl def ensure_translators(): if st.session_state.translators_loaded: return try: tok, mdl = load_ms_en_model() st.session_state.ms_en_tok = tok st.session_state.ms_en_mdl = mdl st.session_state.translators_loaded = True st.session_state.trans_error = "" except Exception as e: st.session_state.trans_error = f"{type(e).__name__}: {e}" st.session_state.translators_loaded = True def translate_ms_en(text: str) -> str: tok = st.session_state.ms_en_tok mdl = st.session_state.ms_en_mdl if tok is None or mdl is None or not text.strip(): return text try: import torch cleaned = normalise_ms(text) for tagged in [f">>ms<< {cleaned}", cleaned]: inputs = tok([tagged], return_tensors="pt", padding=True, truncation=True, max_length=512) with torch.no_grad(): out = mdl.generate(**inputs, num_beams=4, max_length=512) result = tok.decode(out[0], skip_special_tokens=True) if is_valid_translation(result, text): return result return text except Exception: return text def detect_language(text: str) -> str: try: from langdetect import detect return "ms" if detect(text) in ("ms", "id") else "en" except Exception: return "en" # ── Inference ───────────────────────────────────────────────────────────────── def clean_field(s: str) -> str: return s.strip().lstrip(";").strip().lstrip("(").strip().rstrip(")").strip() def parse_output(raw: str): quads = [] for part in raw.split(")"): part = clean_field(part) if not part: continue fields = [clean_field(f) for f in part.split("|")] while len(fields) < 4: fields.append("implicit") quads.append(tuple(fields[:4])) return quads def run_inference(text, tokenizer, model): import torch prompt = f"Extract ACOS quadruples from this course review: {text}" inputs = tokenizer(prompt, return_tensors="pt", max_length=256, truncation=True) with torch.no_grad(): outputs = model.generate(**inputs, max_new_tokens=128, num_beams=4, early_stopping=True) raw = tokenizer.decode(outputs[0], skip_special_tokens=True) return raw, parse_output(raw) def render_table_en(quads): rows = "" for asp, cat, op, sent in quads: s = sent.lower() sc = "positive" if "pos" in s else ("negative" if "neg" in s else "neutral") asp_h = f'{asp}' if asp == "implicit" else asp op_h = f'{op}' if op == "implicit" else op rows += (f"{asp_h}{cat}{op_h}" f"{sent}") return (f'' f'' f'{rows}
AspectCategoryOpinionSentiment
') def render_table_bilingual(quads): sent_bm = {"positive": "positif", "negative": "negatif", "neutral": "neutral"} rows = "" for asp, cat, op, sent in quads: s = sent.lower() sc = "positive" if "pos" in s else ("negative" if "neg" in s else "neutral") if asp == "implicit": asp_cell = 'implicittersirat' else: asp_bm = to_bm(asp) asp_cell = (f'{asp}' + (f'{asp_bm}' if asp_bm != asp else '')) cat_bm = to_bm(cat) cat_cell = (f'{cat}' + (f'{cat_bm}' if cat_bm != cat else '')) if op == "implicit": op_cell = 'implicittersirat' else: op_bm = to_bm(op) op_cell = (f'{op}' + (f'{op_bm}' if op_bm != op else '')) sent_cell = (f"{sent}" f"{sent_bm.get(sc, sent)}") rows += (f"{asp_cell}{cat_cell}" f"{op_cell}{sent_cell}") return (f'' f'' f'' f'{rows}
Aspect / AspekCategory / KategoriOpinion / PendapatSentiment / Sentimen
') def set_review(text: str, label: str = ""): st.session_state.review_text = text st.session_state.last_sample_label = label st.session_state.ta_key += 1 # ── Auto-load model ─────────────────────────────────────────────────────────── if not st.session_state.model_loaded: with st.spinner("Loading ACOS model…"): try: tok, mdl = load_model() st.session_state.tokenizer = tok st.session_state.model = mdl st.session_state.model_loaded = True except Exception as e: st.error(f"Model load failed: {e}") # ════════════════════════════════════════════════════════════ # SIDEBAR # ════════════════════════════════════════════════════════════ with st.sidebar: st.markdown("## 🎓 ACOS System") st.markdown("Aspect-Category-Opinion-Sentiment extraction for course reviews.") st.markdown("---") st.markdown("**Model Status**") if st.session_state.model_loaded: st.success("✓ ACOS model loaded") else: st.warning("Not loaded — check model files") st.markdown("**Language / Bahasa**") st.markdown("🇬🇧 English · 🇲🇾 Bahasa Melayu") st.markdown("---") st.markdown("**Demo Controls**") if st.button("▶ Run All Samples"): st.session_state["run_all"] = True if st.button("🎲 Random Sample"): pick_label, pick_text = random.choice(SAMPLES_EN + SAMPLES_MS) set_review(pick_text, pick_label) st.session_state.last_loaded_en = PLACEHOLDER_EN st.session_state.last_loaded_ms = PLACEHOLDER_MS st.session_state.en_sel_ver += 1 st.session_state.ms_sel_ver += 1 st.rerun() if st.button("✕ Clear"): set_review("", "") st.session_state.last_loaded_en = PLACEHOLDER_EN st.session_state.last_loaded_ms = PLACEHOLDER_MS st.session_state.en_sel_ver += 1 st.session_state.ms_sel_ver += 1 st.session_state.batch_results = [] st.rerun() if st.session_state.get("trans_error"): st.markdown("---") st.markdown("**⚠️ Translation Error**") with st.expander("Show error details"): st.code(st.session_state.trans_error, language="text") if st.button("🔄 Retry"): load_ms_en_model.clear() st.session_state.translators_loaded = False st.session_state.trans_error = "" st.rerun() st.markdown("---") st.markdown("**Model Info**") st.markdown(f"""
ACOS: flan-t5-base
Epochs: 8 · Best F1: 0.2308
Translation: Helsinki-NLP MarianMT
Hub: {HF_MODEL_ID}
""", unsafe_allow_html=True) # ── Header ──────────────────────────────────────────────────────────────────── st.markdown("""
🎓 ACOS Course Evaluation System

Aspect-Category-Opinion-Sentiment structured extraction  ·  🇬🇧 English  ·  🇲🇾 Bahasa Melayu

""", unsafe_allow_html=True) tab_single, tab_batch, tab_perf, tab_about = st.tabs( ["Single Review", "Batch Analysis", "Performance", "About"] ) # ════════════════════════════════════════════════════════════ # TAB 1 · Single Review # ════════════════════════════════════════════════════════════ with tab_single: if st.session_state.model_loaded: st.markdown('
● Model ready — auto-detects English / Bahasa Melayu · bilingual 4-column results for Melayu input
', unsafe_allow_html=True) else: st.error("Model not loaded. Ensure model files are in the same folder as app.py.") st.markdown("**How it works**") st.markdown("""
✍️

1 · Enter a Review

Paste any course review in English or Bahasa Melayu, or pick a built-in sample.

🌐

2 · Language Detection

Bahasa Melayu input is auto-detected and translated to English via Helsinki-NLP MarianMT.

⚙️

3 · ACOS Inference

FLAN-T5-base (fine-tuned on 27,470 quadruples) extracts Aspect · Category · Opinion · Sentiment.

📊

4 · Bilingual Results

4-column ACOS table — each cell shows the English term and its Bahasa Melayu equivalent below.

""", unsafe_allow_html=True) st.markdown("---") col_input, col_samples = st.columns([3, 1]) with col_samples: st.markdown("**Sample Reviews**") st.caption("🇬🇧 English") labels_en = [PLACEHOLDER_EN] + [s[0] for s in SAMPLES_EN] chosen_en = st.selectbox( "EN", labels_en, key=f"sel_en_{st.session_state.en_sel_ver}", label_visibility="collapsed", ) if chosen_en != PLACEHOLDER_EN and chosen_en != st.session_state.last_loaded_en: idx = [s[0] for s in SAMPLES_EN].index(chosen_en) set_review(SAMPLES_EN[idx][1], chosen_en) st.session_state.last_loaded_en = chosen_en st.session_state.last_loaded_ms = PLACEHOLDER_MS st.session_state.ms_sel_ver += 1 st.rerun() st.caption("🇲🇾 Bahasa Melayu") labels_ms = [PLACEHOLDER_MS] + [s[0] for s in SAMPLES_MS] chosen_ms = st.selectbox( "MS", labels_ms, key=f"sel_ms_{st.session_state.ms_sel_ver}", label_visibility="collapsed", ) if chosen_ms != PLACEHOLDER_MS and chosen_ms != st.session_state.last_loaded_ms: idx = [s[0] for s in SAMPLES_MS].index(chosen_ms) set_review(SAMPLES_MS[idx][1], chosen_ms) st.session_state.last_loaded_ms = chosen_ms st.session_state.last_loaded_en = PLACEHOLDER_EN st.session_state.en_sel_ver += 1 st.rerun() if st.session_state.last_sample_label: st.caption(f"Loaded: *{st.session_state.last_sample_label}*") with col_input: review_input = st.text_area( "Course Review (English or Bahasa Melayu)", value=st.session_state.review_text, height=180, placeholder="Paste a course review here…\nTampal ulasan kursus di sini…", key=f"ta_{st.session_state.ta_key}", ) analyse_clicked = st.button("Analyse →", type="primary") if analyse_clicked and review_input.strip(): if not st.session_state.model_loaded: st.error("Model is not loaded.") else: with st.spinner("Detecting language and running inference…"): t0 = time.time() lang = detect_language(review_input.strip()) if lang == "ms": ensure_translators() if st.session_state.trans_error: st.warning("⚠️ Translation failed — see **Translation Error** in sidebar.") translated_en = translate_ms_en(review_input.strip()) inference_text = translated_en else: translated_en = None inference_text = review_input.strip() raw, quads = run_inference( inference_text, st.session_state.tokenizer, st.session_state.model, ) elapsed = time.time() - t0 if lang == "ms": st.markdown('🇲🇾 Bahasa Melayu — bilingual ACOS table', unsafe_allow_html=True) st.markdown(f'
📝 Terjemahan (EN): {translated_en}
', unsafe_allow_html=True) else: st.markdown('🇬🇧 English', unsafe_allow_html=True) st.markdown(f"**{len(quads)} quadruple(s) extracted** — _{elapsed:.2f}s_") if quads: if lang == "ms": st.markdown(render_table_bilingual(quads), unsafe_allow_html=True) else: st.markdown(render_table_en(quads), unsafe_allow_html=True) else: st.info("No valid quadruples parsed from model output.") with st.expander("Raw model output"): st.code(raw) elif analyse_clicked: st.warning("Please enter a review.") # ════════════════════════════════════════════════════════════ # TAB 2 · Batch Analysis # ════════════════════════════════════════════════════════════ with tab_batch: st.markdown("#### Batch Analysis") st.markdown("Upload a CSV/Excel file. Mixed English/Bahasa Melayu rows are handled automatically.") run_all_flag = st.session_state.pop("run_all", False) uploaded = st.file_uploader("Upload CSV or Excel", type=["csv", "xlsx", "xls"], key="batch_upload") if uploaded: st.session_state.upload_filename = sanitize_filename(os.path.splitext(uploaded.name)[0]) col_run, col_demo, _ = st.columns([1, 1, 4]) run_btn = col_run.button("▶ Run Batch", type="primary") demo_btn = col_demo.button("Run Demo Samples") pending_texts = None if demo_btn or run_all_flag: pending_texts = [s[1] for s in SAMPLES_EN + SAMPLES_MS] st.session_state.upload_filename = "demo" elif run_btn and uploaded: try: if uploaded.name.endswith((".xlsx", ".xls")): df_up = pd.read_excel(uploaded) else: raw_bytes = uploaded.read() df_up = None for enc in ["utf-8-sig", "utf-8", "gbk", "gb2312", "latin-1"]: try: df_up = pd.read_csv(io.BytesIO(raw_bytes), encoding=enc) break except Exception: continue if df_up is None: raise ValueError("Unable to decode file.") text_cands = [c for c in df_up.columns if any(k in c.lower() for k in ["text","review","sentence","comment","ulasan"])] default_col = text_cands[0] if text_cands else df_up.select_dtypes(include="object").columns[0] col_sel, _ = st.columns([2, 3]) with col_sel: text_col = st.selectbox("Text column", df_up.columns.tolist(), index=df_up.columns.tolist().index(default_col)) pending_texts = df_up[text_col].dropna().astype(str).tolist() st.caption(f"{len(pending_texts)} rows loaded from **{text_col}**") except Exception as e: st.error(f"Could not read file: {e}") elif uploaded: try: if uploaded.name.endswith((".xlsx", ".xls")): df_up = pd.read_excel(uploaded) else: raw_bytes = uploaded.read() df_up = None for enc in ["utf-8-sig", "utf-8", "gbk", "gb2312", "latin-1"]: try: df_up = pd.read_csv(io.BytesIO(raw_bytes), encoding=enc) break except Exception: continue if df_up is not None: text_cands = [c for c in df_up.columns if any(k in c.lower() for k in ["text","review","sentence","comment","ulasan"])] default_col = text_cands[0] if text_cands else df_up.select_dtypes(include="object").columns[0] col_sel, _ = st.columns([2, 3]) with col_sel: st.selectbox("Text column", df_up.columns.tolist(), index=df_up.columns.tolist().index(default_col)) st.caption(f"{len(df_up)} rows — click **Run Batch** to analyse") except Exception: pass if pending_texts: if not st.session_state.model_loaded: st.error("Model is not loaded.") else: ensure_translators() results = [] prog = st.progress(0, text="Running batch…") for i, txt in enumerate(pending_texts): lang = detect_language(txt) txt_en = translate_ms_en(txt) if lang == "ms" else txt raw, quads = run_inference(txt_en, st.session_state.tokenizer, st.session_state.model) sent_bm_map = {"positive":"positif","negative":"negatif","neutral":"neutral"} for q in quads: s = q[3].lower() sc = "positive" if "pos" in s else ("negative" if "neg" in s else "neutral") if lang == "ms": asp_bm = "tersirat" if q[0]=="implicit" else to_bm(q[0]) op_bm = "tersirat" if q[2]=="implicit" else to_bm(q[2]) cat_bm = to_bm(q[1]) else: asp_bm = op_bm = cat_bm = "" results.append({ "Review": txt[:80] + ("…" if len(txt) > 80 else ""), "Language": "BM→EN" if lang == "ms" else "EN", "Aspect (EN)": q[0], "Aspek (BM)": asp_bm, "Category (EN)": q[1], "Kategori (BM)": cat_bm, "Opinion (EN)": q[2], "Pendapat (BM)": op_bm, "Sentiment": q[3], "Sentimen": sent_bm_map.get(sc,"") if lang=="ms" else "", }) prog.progress((i+1)/len(pending_texts), text=f"{i+1}/{len(pending_texts)}") prog.empty() st.session_state.batch_results = results if st.session_state.batch_results: df_res = pd.DataFrame(st.session_state.batch_results) def colour_sentiment(val): v = str(val).lower() if "pos" in v: return "color:#16a34a; font-weight:600" if "neg" in v: return "color:#dc2626; font-weight:600" return "color:#64748b" st.dataframe(df_res.style.map(colour_sentiment, subset=["Sentiment"]), use_container_width=True, height=420, hide_index=True) fname_stem = st.session_state.get("upload_filename", "batch") download_name = f"{fname_stem}_ACOS_Analysis.csv" st.download_button("⬇ Download CSV", data=df_res.to_csv(index=False).encode("utf-8-sig"), file_name=download_name, mime="text/csv") st.caption(f"Saved as: **{download_name}**") # ════════════════════════════════════════════════════════════ # TAB 3 · Performance # ════════════════════════════════════════════════════════════ with tab_perf: st.markdown("#### Model Training Performance") df_h = pd.DataFrame(HISTORY) st.markdown("""
0.2308
Best F1 (Epoch 8)
0.2420
Precision
0.2205
Recall
8
Epochs Trained
""", unsafe_allow_html=True) col_l, col_r = st.columns(2) with col_l: st.markdown("**Training Loss**") st.line_chart(df_h.set_index("Epoch")["Train Loss"]) with col_r: st.markdown("**Validation F1**") st.line_chart(df_h.set_index("Epoch")["Val F1"]) st.markdown("**Full Training History**") st.dataframe( df_h.style.highlight_max(subset=["Val F1","Precision","Recall"], color="#dcfce7") .highlight_min(subset=["Train Loss"], color="#dcfce7"), use_container_width=True, hide_index=True, ) st.markdown("**Baseline Comparison**") df_comp = pd.DataFrame({ "Model": ["Random Baseline", "TF-IDF k-NN", "FLAN-T5-base (Fine-tuned)"], "F1 Score": [0.0000, 0.0579, 0.2308], "Precision": ["N/A", "N/A", "0.2420"], "Recall": ["N/A", "N/A", "0.2205"], }) st.dataframe(df_comp, use_container_width=True, hide_index=True) st.caption("Precision and Recall are not reported for the baselines as they were not measured in the original evaluation.") # ════════════════════════════════════════════════════════════ # TAB 4 · About # ════════════════════════════════════════════════════════════ with tab_about: st.markdown(""" #### About This System **ACOS Course Evaluation System** is a natural-language processing research prototype developed as part of FYP1 (WIA3002) at the University of Malaya. The system performs structured sentiment analysis on student course reviews by extracting **Aspect-Category-Opinion-Sentiment (ACOS) quadruples** — a fine-grained representation that captures *what* is being discussed, *how* it is categorised, *what opinion* is expressed, and the overall *sentiment polarity*. --- #### Task Definition Given a free-text course review, the system extracts one or more quadruples of the form: > **(Aspect, Category, Opinion, Sentiment)** | Field | Description | Example | |-------|-------------|---------| | **Aspect** | The specific entity discussed (may be implicit) | *lecture notes*, *implicit* | | **Category** | The high-level topic category | *presentation quality*, *faculty general* | | **Opinion** | The opinion expression used (may be implicit) | *completely disorganised*, *implicit* | | **Sentiment** | Overall polarity of the opinion | *negative*, *positive*, *neutral* | --- #### Model | Property | Detail | |----------|--------| | Base model | `google/flan-t5-base` (250M parameters) | | Fine-tuning task | Sequence-to-sequence ACOS quadruple generation | | Input prompt | `Extract ACOS quadruples from this course review: {text}` | | Output format | `(aspect \| category \| opinion \| sentiment)` | | Training data | OATS-ABSA — 27,470 annotated quadruples across Amazon, Coursera, and Hotels domains | | Training setup | 8 epochs · AdaFactor optimiser · lr = 5×10⁻⁵ · batch size 8 · Kaggle T4×2 GPU | | Best checkpoint | **Epoch 8** · F1 = 0.2308 · Precision = 0.2420 · Recall = 0.2205 | | Evaluation | Exact Match F1 — all four fields must match exactly to count as correct | The model is hosted on HuggingFace Hub: [`lijinzheyy/ACOS_CourseEvaluation_System`](https://huggingface.co/lijinzheyy/ACOS_CourseEvaluation_System) --- #### Baseline Comparison | Model | F1 Score | Notes | |-------|----------|-------| | Random Baseline | 0.0000 | Random quadruple sampling from training set | | TF-IDF + k-NN | 0.0579 | Nearest-neighbour retrieval (k=1, cosine similarity) | | **FLAN-T5-base (Fine-tuned)** | **0.2308** | Best checkpoint, Epoch 8 | The fine-tuned model achieves **3.98×** higher F1 than the strongest baseline. --- #### Language Pipeline This system supports both **English** and **Bahasa Melayu** input through an automated bilingual pipeline: | Step | Detail | |------|--------| | 1. Detection | `langdetect` identifies Bahasa Melayu (`ms`/`id`) vs English (`en`) | | 2. Translation (BM→EN) | `Helsinki-NLP/opus-mt-mul-en` (MarianMT) translates Malay input to English | | 3. ACOS Inference | Fine-tuned FLAN-T5 extracts quadruples from English text | | 4. Bilingual Display | Results shown in 4-column table with English terms and Bahasa Melayu equivalents | > **Note:** The MS→EN translation model (~300 MB) is downloaded automatically on first Bahasa Melayu input and cached thereafter. Translation quality depends on the MarianMT model; Bahasa Melayu ACOS accuracy is lower than English as a result. --- #### Limitations - The ACOS model is trained on English text only — Bahasa Melayu accuracy is dependent on upstream translation quality. - Exact Match F1 is a strict metric; partial quadruple matches score zero regardless of how many fields are correct. - The bilingual table uses a curated static dictionary; novel opinion words outside the dictionary are displayed in English only. - This is a research prototype and has not been validated for production deployment. --- #### Tech Stack `Python 3.10` · `Streamlit` · `HuggingFace Transformers` · `PyTorch` · `Pandas` · `langdetect` · `Helsinki-NLP/opus-mt-mul-en` (MarianMT) --- """)