"""Feature engineering for the fediverse bot classifier. Feature layout fed to the model: [scaled numeric | language one-hot | bio embedding]. fastText is imported lazily and its language-ID model is downloaded on first use. """ import json import os import urllib.request from functools import lru_cache from pathlib import Path import joblib import numpy as np import pandas as pd HERE = Path(__file__).resolve().parent NUMERIC_COLS = [ 'statuses_count', 'followers_count', 'following_count', 'username_length', 'username_num_digits', 'username_num_letters', 'username_num_special', 'username_starts_with_digit', 'username_ends_with_digit', ] UNKNOWN_LANG = 'unknown' OTHER_LANG = 'other' MODEL_PATH = HERE / 'logistic_regression_model.joblib' SCALER_PATH = HERE / 'scaler.joblib' LANG_VOCAB_PATH = HERE / 'language_vocab.json' # fastText lid.176 (CC-BY-SA 3.0, Facebook) is not redistributed here; it is fetched on first use. FASTTEXT_URL = 'https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz' FASTTEXT_MODEL_PATH = Path(os.getenv( 'FASTTEXT_MODEL_PATH', Path.home() / '.cache' / 'fediverse-bot-detector' / 'lid.176.ftz', )) def extract_username_features(username): username = username or '' return { 'username_length': len(username), 'username_num_digits': sum(c.isdigit() for c in username), 'username_num_letters': sum(c.isalpha() for c in username), 'username_num_special': sum(not c.isalnum() for c in username), 'username_starts_with_digit': int(bool(username) and username[0].isdigit()), 'username_ends_with_digit': int(bool(username) and username[-1].isdigit()), } def _ensure_fasttext_model(): if not FASTTEXT_MODEL_PATH.exists(): FASTTEXT_MODEL_PATH.parent.mkdir(parents=True, exist_ok=True) tmp = FASTTEXT_MODEL_PATH.with_suffix('.part') urllib.request.urlretrieve(FASTTEXT_URL, tmp) tmp.replace(FASTTEXT_MODEL_PATH) return FASTTEXT_MODEL_PATH @lru_cache(maxsize=1) def _fasttext_model(): import fasttext return fasttext.load_model(str(_ensure_fasttext_model())) def normalize_lang(lang): if not isinstance(lang, str) or not lang.strip(): return UNKNOWN_LANG return lang.replace('__label__', '').strip().lower() def detect_language(text): """fastText lid.176 language code of a bio; 'unknown' when empty.""" if not isinstance(text, str) or not text.strip(): return UNKNOWN_LANG # model.f.predict avoids the numpy-2 incompatibility of FastText.predict preds = _fasttext_model().f.predict(text.replace('\n', ' '), 1, 0.0, 'strict') return normalize_lang(preds[0][1]) if preds else UNKNOWN_LANG def language_onehot(languages, vocab): """One-hot over vocab + OTHER_LANG. Languages outside vocab map to 'other'.""" columns = list(vocab) + [OTHER_LANG] index = {lang: i for i, lang in enumerate(columns)} out = np.zeros((len(languages), len(columns)), dtype=np.float32) for row, lang in enumerate(languages): out[row, index.get(normalize_lang(lang), index[OTHER_LANG])] = 1.0 return out def add_account_features(df, text_col='clean_note'): """Add the username and language columns.""" df = df.copy() missing = [c for c in NUMERIC_COLS[3:9] if c not in df.columns] if missing: username_features = df['username'].apply(extract_username_features).apply(pd.Series) df = pd.concat([df.drop(columns=missing, errors='ignore'), username_features], axis=1) if 'language' not in df.columns: df['language'] = [detect_language(t) for t in df[text_col]] return df def load_artifacts(): with open(LANG_VOCAB_PATH) as f: vocab = json.load(f) return joblib.load(MODEL_PATH), joblib.load(SCALER_PATH), vocab def build_feature_matrix(df, embeddings, scaler, vocab): """[scaled numeric | language one-hot | embeddings]; df must already have all features.""" numeric = scaler.transform(df[NUMERIC_COLS].astype(float).values) return np.hstack((numeric, language_onehot(df['language'].tolist(), vocab), embeddings))