#!/usr/bin/env python3 """Train and benchmark the SenseCatch sentiment models. Trains the four deployed models (Naive Bayes, Logistic Regression, LinearSVC, NBSVM) on the NLTK movie reviews plus the IMDB training split. Can also train IMDB-only models used as benchmark references. Tuning uses a held-out dev set; the IMDB test set is only touched for final scoring. Preprocessing reuses the serving path so training matches serving. Usage: python src/training/retrain.py # report the corpus python src/training/retrain.py --imdb-benchmark # IMDB-only benchmark models python src/training/retrain.py --train-deployed # the four deployed models python src/training/retrain.py --manifest # write the training manifest """ import os import sys THIS_DIR = os.path.dirname(os.path.abspath(__file__)) # .../src/training PROJECT_ROOT = os.path.dirname(os.path.dirname(THIS_DIR)) # repo root for _p in (PROJECT_ROOT, THIS_DIR): if _p not in sys.path: sys.path.insert(0, _p) import data_split # noqa: E402 (the seeded train/dev/test split) class CorpusError(RuntimeError): """Raised when a declared training corpus is missing or empty.""" def load_nltk_movie_reviews(): """Full NLTK movie_reviews corpus -> (texts, labels); label 1=pos / 0=neg. FAIL LOUDLY if missing.""" import nltk for cand in (os.path.join(PROJECT_ROOT, "nltk_data"), os.path.join(PROJECT_ROOT, "artifacts", "nltk_data")): if os.path.isdir(cand) and cand not in nltk.data.path: nltk.data.path.insert(0, cand) try: from nltk.corpus import movie_reviews fileids = movie_reviews.fileids() except LookupError as e: raise CorpusError( "NLTK 'movie_reviews' corpus is unavailable. Run " "`python src/sensecatch/utils/nltk_downloader.py` or fix the nltk_data path. " f"Original: {e}" ) from e if not fileids: raise CorpusError("NLTK 'movie_reviews' returned 0 fileids; the corpus is empty.") texts, labels = [], [] for cat in movie_reviews.categories(): label = 1 if cat == "pos" else 0 for fid in movie_reviews.fileids(cat): texts.append(" ".join(movie_reviews.words(fid))) labels.append(label) return texts, labels def _require_imdb_dir(): """Raise CorpusError unless datasets/aclImdb/train/{pos,neg}/ exist and are non-empty.""" for sub in ("pos", "neg"): d = os.path.join(data_split.IMDB_DIR, "train", sub) if not os.path.isdir(d) or not os.listdir(d): raise CorpusError( f"IMDB training data missing/empty: {d}. " "Expected datasets/aclImdb/train/{pos,neg}/. Download and extract the IMDB dataset." ) def load_imdb_train(): """IMDB-train from the seeded 'train' split (22,500) -> (texts, labels). FAIL LOUDLY if missing.""" _require_imdb_dir() items = data_split.get_split()["train"] if not items: raise CorpusError("IMDB 'train' split is empty.") return data_split.load_texts(items) def load_imdb_dev(): """Held-out IMDB 'dev' set (2,500) for tuning. (TEST is never loaded here.)""" _require_imdb_dir() return data_split.load_texts(data_split.get_split()["dev"]) def load_imdb_only_train(): """IMDB training split only, as (texts, labels). Used for the benchmark models.""" return load_imdb_train() def load_training_corpus(): """GENERAL training corpus = NLTK movie_reviews + IMDB-train. NO Twitter. FAIL LOUDLY. Returns {'texts','labels','sources','nltk_n','imdb_n','n'}. """ nltk_texts, nltk_labels = load_nltk_movie_reviews() imdb_texts, imdb_labels = load_imdb_train() texts = nltk_texts + imdb_texts labels = nltk_labels + imdb_labels sources = ["nltk"] * len(nltk_texts) + ["imdb"] * len(imdb_texts) if not texts: raise CorpusError("Combined training corpus is empty.") return { "texts": texts, "labels": labels, "sources": sources, "nltk_n": len(nltk_texts), "imdb_n": len(imdb_texts), "n": len(texts), } def preprocess(texts, ensemble=None): """Preprocess texts with the serving path (ensemble._preprocess_text), cached on disk. Reuses evaluate.preprocess_texts so training and evaluation share one cache. Slow on a cold cache (NLTK pos_tag and ne_chunk run per text). """ import evaluate as ev if ensemble is None: ensemble = ev.load_ensemble() return ev.preprocess_texts(ensemble, texts) # IMDB-only benchmark models (reference numbers, not the deployed models). # Trained on the IMDB training split, tuned on the dev set, scored once on the # IMDB test set. Plain sklearn vectorizers on raw text, so it runs fast. BENCHMARK_PATH = os.path.join(PROJECT_ROOT, "artifacts", "benchmark_imdb_only.json") _C_GRID = [0.1, 1.0, 10.0] def _wilson_ci(acc, n, z=1.96): import math if n == 0: return (0.0, 0.0) denom = 1 + z * z / n center = (acc + z * z / (2 * n)) / denom half = (z * math.sqrt(acc * (1 - acc) / n + z * z / (4 * n * n))) / denom return (center - half, center + half) def _mcnemar(y_true, pred_a, pred_b): """McNemar's test, model_b vs model_a. Returns (a_only_correct, b_only_correct, chi2, p).""" import numpy as np from scipy.stats import chi2 as _chi2 yt, pa, pb = np.asarray(y_true), np.asarray(pred_a), np.asarray(pred_b) ca, cb = (pa == yt), (pb == yt) b = int(np.sum(ca & ~cb)) # a correct, b wrong c = int(np.sum(~ca & cb)) # a wrong, b correct stat = (abs(b - c) - 1) ** 2 / (b + c) if (b + c) > 0 else 0.0 return b, c, float(stat), float(_chi2.sf(stat, 1)) def _metrics(name, y_true, y_pred): from sklearn.metrics import accuracy_score, f1_score acc = float(accuracy_score(y_true, y_pred)) lo, hi = _wilson_ci(acc, len(y_true)) return {"model": name, "accuracy": round(acc, 4), "acc_ci95": [round(lo, 4), round(hi, 4)], "macro_f1": round(float(f1_score(y_true, y_pred, average="macro")), 4), "n_test": len(y_true)} def _tune_C_on_dev(make_model, Xtr, ytr, Xdev, ydev, grid): """Fit make_model(C) on TRAIN, pick the C with best DEV accuracy (no test peeking).""" from sklearn.metrics import accuracy_score best_C, best_acc = grid[0], -1.0 for C in grid: m = make_model(C); m.fit(Xtr, ytr) a = accuracy_score(ydev, m.predict(Xdev)) if a > best_acc: best_acc, best_C = a, C return best_C, best_acc def benchmark_tfidf_linear(kind, tr_texts, ytr, dev_texts, ydev, te_texts, yte): """IMDB-only tuned TF-IDF linear model. kind in {'logreg','linsvc'}. Levers vs the OLD pipeline: max_features=50000 (was 10000) and NO stopwords (was 'english').""" from sklearn.feature_extraction.text import TfidfVectorizer from sklearn.linear_model import LogisticRegression from sklearn.svm import LinearSVC vec = TfidfVectorizer(max_features=50000, ngram_range=(1, 2), min_df=3, sublinear_tf=True, stop_words=None, strip_accents="unicode") Xtr = vec.fit_transform(tr_texts); Xdev = vec.transform(dev_texts); Xte = vec.transform(te_texts) if kind == "logreg": make = lambda C: LogisticRegression(C=C, max_iter=1000, solver="liblinear") name = "IMDB-only TF-IDF LogisticRegression (tuned, 50k feats, no stopwords)" else: make = lambda C: LinearSVC(C=C, max_iter=4000) name = "IMDB-only TF-IDF LinearSVC (tuned, 50k feats, no stopwords)" bestC, devacc = _tune_C_on_dev(make, Xtr, ytr, Xdev, ydev, _C_GRID) final = make(bestC); final.fit(Xtr, ytr) pred = final.predict(Xte) m = _metrics(name, yte, pred) m.update({"tuned_C": bestC, "dev_acc": round(devacc, 4), "n_features": Xtr.shape[1]}) return m, pred def benchmark_nbsvm(tr_texts, ytr, dev_texts, ydev, te_texts, yte, beta=0.25, use_interpolation=False): """Faithful NBSVM (Wang & Manning 2012), IMDB-only: binarized bigrams, NO stopword removal, punctuation kept (token_pattern=[^\\s]+); NB log-count-ratio r from TRAIN counts only, applied to test; LinearSVC on r-weighted features. (beta interpolation optional, off by default.)""" import numpy as np from sklearn.feature_extraction.text import CountVectorizer from sklearn.svm import LinearSVC from sklearn.metrics import accuracy_score vec = CountVectorizer(binary=True, ngram_range=(1, 2), lowercase=True, token_pattern=r"[^\s]+", min_df=5) Xtr = vec.fit_transform(tr_texts); Xdev = vec.transform(dev_texts); Xte = vec.transform(te_texts) y = np.asarray(ytr) p = 1.0 + np.asarray(Xtr[y == 1].sum(axis=0)).ravel() # smoothed positive feature counts (TRAIN only) q = 1.0 + np.asarray(Xtr[y == 0].sum(axis=0)).ravel() # smoothed negative feature counts (TRAIN only) r = np.log((p / p.sum()) / (q / q.sum())).reshape(1, -1) nb = lambda X: X.multiply(r).tocsr() # binarized presence * r Xtr_nb, Xdev_nb, Xte_nb = nb(Xtr), nb(Xdev), nb(Xte) devacc, bestC_val = -1.0, 1.0 for C in [0.5, 1.0, 5.0]: s = LinearSVC(C=C, max_iter=4000); s.fit(Xtr_nb, y) a = accuracy_score(ydev, s.predict(Xdev_nb)) if a > devacc: devacc, bestC_val = a, C svm = LinearSVC(C=bestC_val, max_iter=4000); svm.fit(Xtr_nb, y) pred = svm.predict(Xte_nb) name = "IMDB-only NBSVM (faithful: binarized bigrams, no stopwords, r from train; LinearSVC)" m = _metrics(name, yte, pred) m.update({"tuned_C": bestC_val, "dev_acc": round(devacc, 4), "n_features": Xtr.shape[1], "beta_interpolation": (beta if use_interpolation else None)}) return m, pred def run_imdb_only_benchmark(): """Train + evaluate the IMDB-only BENCHMARK models on IMDB TEST once. Saves JSON. REFERENCE only. These are NOT the deployed app models (that is 7.7); the app .pkl contract is untouched here.""" import json import time print("=== IMDB-only benchmark (reference numbers, not deployed) ===") t0 = time.time() tr_texts, ytr = load_imdb_only_train() dev_texts, ydev = load_imdb_dev() te_texts, yte = data_split.load_texts(data_split.get_split()["test"]) print(f"train {len(ytr)} / dev {len(ydev)} / test {len(yte)} (raw text; no _preprocess_text)") results, preds = [], {} for kind in ("logreg", "linsvc"): m, pred = benchmark_tfidf_linear(kind, tr_texts, ytr, dev_texts, ydev, te_texts, yte) results.append(m); preds[m["model"]] = pred print(f" {m['accuracy']} CI{m['acc_ci95']} F1 {m['macro_f1']} C={m['tuned_C']} | {m['model']}") m, pred = benchmark_nbsvm(tr_texts, ytr, dev_texts, ydev, te_texts, yte) results.append(m); preds[m["model"]] = pred print(f" {m['accuracy']} CI{m['acc_ci95']} F1 {m['macro_f1']} C={m['tuned_C']} | {m['model']}") comparisons = [] ordered = sorted(results, key=lambda r: r["accuracy"]) # McNemar each vs the next-best below it for i in range(1, len(ordered)): a, b = ordered[i - 1]["model"], ordered[i]["model"] bb, cc, chi2, pval = _mcnemar(yte, preds[a], preds[b]) comparisons.append({"comparison": f"{b} vs {a}", "b_only_correct": cc, "a_only_correct": bb, "chi2": round(chi2, 2), "p_value": float(f"{pval:.3g}"), "b_significantly_better": bool(pval < 0.05 and cc > bb)}) best = max(results, key=lambda r: r["accuracy"]) out = {"track": "IMDB-only benchmark (reference, NOT deployed)", "test_n": len(yte), "frontier_target": "~88-91% (Wang & Manning 2012 NBSVM ~91.2%)", "models": results, "mcnemar": comparisons, "best_model": best["model"], "best_accuracy": best["accuracy"], "best_acc_ci95": best["acc_ci95"], "runtime_sec": round(time.time() - t0, 1)} os.makedirs(os.path.dirname(BENCHMARK_PATH), exist_ok=True) with open(BENCHMARK_PATH, "w") as f: json.dump(out, f, indent=2) print(f"BEST IMDB-only: {best['accuracy']} (CI {best['acc_ci95']}) -> {best['model']}") print(f"Saved -> {BENCHMARK_PATH} | runtime {out['runtime_sec']}s") return out # The four deployed models shown in the app dropdown. Each is wrapped in # CalibratedClassifierCV (cv=5) so it can output probabilities. Trained on the # NLTK plus IMDB corpus with the serving preprocessing. Saved as a 3-tuple # (model, text_vectorizer, dict_vectorizer) that the app loads. MODELS_DIR = os.path.join(PROJECT_ROOT, "models") DEPLOY_BACKUP = os.path.join(PROJECT_ROOT, "artifacts", "models_backup_pre-step7-deploy") DEPLOY_RESULTS = os.path.join(PROJECT_ROOT, "artifacts", "deploy_results.json") _STALE_VECTORIZER_FILES = ( "count_vectorizer.pkl", "tfidf_vectorizer.pkl", "dict_vectorizer.pkl", "feature_dimensions.pkl", "feature_info.txt", ) def backup_current_models(): """Copy the current model files to the backup folder, with their SHA-256 hashes.""" import shutil, hashlib, glob, json os.makedirs(DEPLOY_BACKUP, exist_ok=True) hashes = {} files = sorted(glob.glob(os.path.join(MODELS_DIR, "*.pkl"))) + sorted(glob.glob(os.path.join(MODELS_DIR, "*.txt"))) for f in files: shutil.copy2(f, DEPLOY_BACKUP) with open(f, "rb") as fh: hashes[os.path.basename(f)] = hashlib.sha256(fh.read()).hexdigest() with open(os.path.join(DEPLOY_BACKUP, "HASHES.json"), "w") as fh: json.dump(hashes, fh, indent=2) return DEPLOY_BACKUP, hashes def _remove_stale_vectorizer_files(): """Remove old standalone vectorizer files if present. They would override the vectorizers saved with each model and cause silent feature-size mismatches.""" removed = [] for name in _STALE_VECTORIZER_FILES: p = os.path.join(MODELS_DIR, name) if os.path.exists(p): os.remove(p); removed.append(name) return removed def _lexicon_dicts(lexicon, processed): """9-dim lexicon dict features; negatives clamped to 0 (MNB needs non-negative).""" out = [] for t in processed: d = lexicon.extract_all_features(t) for k, v in list(d.items()): if isinstance(v, (int, float)) and v < 0: d[k] = 0.0 out.append(d) return out def train_deployed_models(n_sample=None): """Train + save the FOUR deployed GENERAL models (NB, LR, LinearSVC, NBSVM), all calibrated. n_sample != None => DRY-RUN on a tiny subset (small vocab) to validate the pipeline fast. Writes artifacts/deploy_results.json and returns the results dict.""" import json, time, pickle, random import numpy as np from scipy.sparse import hstack from sklearn.feature_extraction.text import CountVectorizer, TfidfVectorizer from sklearn.feature_extraction import DictVectorizer from sklearn.pipeline import Pipeline from sklearn.naive_bayes import MultinomialNB from sklearn.linear_model import LogisticRegression from sklearn.svm import LinearSVC from sklearn.calibration import CalibratedClassifierCV from sklearn.metrics import accuracy_score import evaluate as ev from src.sensecatch.nbsvm_transformer import NBLogCountRatio t0 = time.time() backup_current_models() corpus = load_training_corpus() tr_texts, ytr = corpus["texts"], list(corpus["labels"]) dev_texts, ydev = load_imdb_dev() if n_sample: idx = list(range(len(tr_texts))); random.Random(0).shuffle(idx); idx = idx[:n_sample] tr_texts = [tr_texts[i] for i in idx]; ytr = [ytr[i] for i in idx] k = max(60, n_sample // 4); dev_texts, ydev = dev_texts[:k], ydev[:k] ytr = np.asarray(ytr); ydev = np.asarray(ydev) ensemble = ev.load_ensemble() print(f"[deploy] preprocessing {len(tr_texts)} train + {len(dev_texts)} dev via serving path " f"({'DRY-RUN' if n_sample else 'FULL; cold cache ~40min'})...", flush=True) tr_proc = ev.preprocess_texts(ensemble, tr_texts) dev_proc = ev.preprocess_texts(ensemble, dev_texts) lex = ensemble.lexicon dv = DictVectorizer() Ltr = dv.fit_transform(_lexicon_dicts(lex, tr_proc)) Ldev = dv.transform(_lexicon_dicts(lex, dev_proc)) mf = 2000 if n_sample else 30000 mindf_nbsvm = 2 if n_sample else 5 specs = [ ("naive_bayes", CountVectorizer(max_features=mf, ngram_range=(1, 2), min_df=2, stop_words=None), (lambda: MultinomialNB(alpha=0.1)), "naive_bayes.pkl"), ("logistic_regression", TfidfVectorizer(max_features=mf, ngram_range=(1, 2), min_df=2, sublinear_tf=True, stop_words=None), (lambda: LogisticRegression(C=10.0, max_iter=1000, solver="liblinear")), "logistic_regression.pkl"), ("linear_svc", TfidfVectorizer(max_features=mf, ngram_range=(1, 2), min_df=2, sublinear_tf=True, stop_words=None), (lambda: LinearSVC(C=1.0, max_iter=4000)), "linear_svc.pkl"), ("nbsvm", Pipeline([("cv", CountVectorizer(binary=True, ngram_range=(1, 2), token_pattern=r"[^\s]+", min_df=mindf_nbsvm)), ("nb", NBLogCountRatio())]), (lambda: LinearSVC(C=0.5, max_iter=4000)), "nbsvm.pkl"), ] results, dev_probas = {}, {} for name, tv, make_base, fname in specs: Xtr_t = tv.fit_transform(tr_proc, ytr) if isinstance(tv, Pipeline) else tv.fit_transform(tr_proc) Xdev_t = tv.transform(dev_proc) Xtr = hstack([Xtr_t, Ltr]).tocsr(); Xdev = hstack([Xdev_t, Ldev]).tocsr() clf = CalibratedClassifierCV(make_base(), cv=5) clf.fit(Xtr, ytr) dev_acc = float(accuracy_score(ydev, clf.predict(Xdev))) with open(os.path.join(MODELS_DIR, fname), "wb") as f: pickle.dump((clf, tv, dv), f) results[name] = {"dev_accuracy": round(dev_acc, 4), "n_features": int(Xtr.shape[1]), "file": fname} if name in ("naive_bayes", "logistic_regression"): dev_probas[name] = clf.predict_proba(Xdev) print(f"[deploy] {name}: dev_acc {dev_acc:.4f} ({Xtr.shape[1]} feats) -> models/{fname}", flush=True) # re-tune NB+LR ensemble weights on DEV only best_w, best_acc = 0.6, -1.0 for w in [0.3, 0.4, 0.5, 0.6, 0.7]: comb = w * dev_probas["naive_bayes"] + (1 - w) * dev_probas["logistic_regression"] a = float(accuracy_score(ydev, np.argmax(comb, axis=1))) if a > best_acc: best_acc, best_w = a, w removed = _remove_stale_vectorizer_files() out = {"track": "deployed general (4 calibrated models)", "dry_run": bool(n_sample), "n_train": int(len(ytr)), "n_dev": int(len(ydev)), "models": results, "ensemble_weights": {"naive_bayes": best_w, "logistic_regression": round(1 - best_w, 2)}, "ensemble_dev_acc": round(best_acc, 4), "backup_dir": DEPLOY_BACKUP, "stale_files_removed": removed, "runtime_sec": round(time.time() - t0, 1)} os.makedirs(os.path.dirname(DEPLOY_RESULTS), exist_ok=True) with open(DEPLOY_RESULTS, "w") as f: json.dump(out, f, indent=2) print(f"[deploy] ensemble weights (dev-tuned): NB {best_w} / LR {round(1-best_w,2)} (dev acc {best_acc:.4f})") print(f"[deploy] saved 4 models; stale vectorizer files removed: {removed or 'none'}; " f"results -> {DEPLOY_RESULTS}; runtime {out['runtime_sec']}s") return out def write_training_manifest(path=None): """Write a manifest describing how the four deployed models were trained. Reads the existing result files and hashes the model files. Does not retrain. Output: artifacts/training_manifest.json.""" import json, hashlib, sys from datetime import datetime from importlib.metadata import version as _ver path = path or os.path.join(PROJECT_ROOT, "artifacts", "training_manifest.json") deploy = json.load(open(DEPLOY_RESULTS)) split = json.load(open(os.path.join(PROJECT_ROOT, "artifacts", "data_split_manifest.json"))) try: import evaluate as _ev preprocess_version = getattr(_ev, "PREPROCESS_VERSION", "v1") except Exception: preprocess_version = "v1" def _sha256(fp): h = hashlib.sha256() with open(fp, "rb") as fh: for chunk in iter(lambda: fh.read(1 << 20), b""): h.update(chunk) return h.hexdigest() def _v(pkg): try: return _ver(pkg) except Exception: return "unknown" imdb_train = split["splits"]["train"]["n"] # 22500 (seeded IMDB-train split) train_total = deploy["n_train"] # 24500 = NLTK + IMDB-train recipes = { "naive_bayes": "CountVectorizer(max_features=30000, ngram(1,2), min_df=2, no stopwords) + lexicon DictVectorizer(9) -> MultinomialNB(alpha=0.1), CalibratedClassifierCV(cv=5)", "logistic_regression": "TfidfVectorizer(max_features=30000, ngram(1,2), min_df=2, sublinear_tf, no stopwords) + lexicon DictVectorizer(9) -> LogisticRegression(C=10.0, liblinear), CalibratedClassifierCV(cv=5)", "linear_svc": "TfidfVectorizer(max_features=30000, ngram(1,2), min_df=2, sublinear_tf, no stopwords) + lexicon DictVectorizer(9) -> LinearSVC(C=1.0), CalibratedClassifierCV(cv=5)", "nbsvm": "Pipeline(CountVectorizer(binary, ngram(1,2), token=non-whitespace runs, min_df=5) -> NBLogCountRatio) + lexicon DictVectorizer(9) -> LinearSVC(C=0.5), CalibratedClassifierCV(cv=5)", } models = {} for name, info in deploy["models"].items(): fp = os.path.join(MODELS_DIR, info["file"]) models[name] = { "file": "models/" + info["file"], "sha256": _sha256(fp) if os.path.exists(fp) else "missing", "n_features": info["n_features"], "dev_accuracy": info["dev_accuracy"], "recipe": recipes.get(name, ""), } manifest = { "generated_at": datetime.now().isoformat(), "seed": 42, "dev_fraction": 0.10, "preprocess_version": preprocess_version, "corpus": { "train_total": train_total, "dev_total": deploy["n_dev"], "sources": { "nltk_movie_reviews": train_total - imdb_train, "imdb_train": imdb_train, }, "balance": "50/50 pos/neg", "twitter": False, }, "data_split": { "seed": split["seed"], "dev_fraction": split["dev_fraction"], "splits": split["splits"], }, "library_versions": { "python": sys.version.split()[0], "scikit-learn": _v("scikit-learn"), "numpy": _v("numpy"), "scipy": _v("scipy"), "nltk": _v("nltk"), }, "models": models, "ensemble_weights": deploy.get("ensemble_weights"), "ensemble_dev_acc": deploy.get("ensemble_dev_acc"), "notes": "DEV accuracies are on the held-out IMDB dev set, not the final test set.", } os.makedirs(os.path.dirname(path), exist_ok=True) with open(path, "w") as f: json.dump(manifest, f, indent=2) print(f"[manifest] wrote {path}") return manifest if __name__ == "__main__": if "--imdb-benchmark" in sys.argv: run_imdb_only_benchmark(); sys.exit(0) if "--dry-run-deploy" in sys.argv: print("DRY-RUN:", train_deployed_models(n_sample=300)); sys.exit(0) if "--train-deployed" in sys.argv: print("DEPLOY:", train_deployed_models()); sys.exit(0) if "--manifest" in sys.argv: write_training_manifest(); sys.exit(0) print("=== SenseCatch retrain.py - corpus foundation (no training) ===") corpus = load_training_corpus() npos = sum(corpus["labels"]) print(f"GENERAL training corpus: {corpus['n']} docs ({npos} pos / {corpus['n'] - npos} neg)") print(f" - NLTK movie_reviews : {corpus['nltk_n']}") print(f" - IMDB-train (seeded): {corpus['imdb_n']}") dev_t, dev_l = load_imdb_dev() print(f"DEV (held-out, for tuning): {len(dev_l)} ({sum(dev_l)} pos / {len(dev_l) - sum(dev_l)} neg)") print("TEST split is NOT loaded here (touched once, at final eval).") print("NO Twitter. Preprocessing = ensemble._preprocess_text (cached, train==serve).")