fediverse-bot-detector / bot_features.py
FranUPM's picture
Initial release: logistic regression fediverse bot detector
8793683 verified
Raw History Blame Contribute Delete
4.13 kB
"""Feature engineering for the fediverse bot classifier.
Feature layout fed to the model: [scaled numeric | language one-hot | bio embedding].
fastText is imported lazily and its language-ID model is downloaded on first use.
"""
import json
import os
import urllib.request
from functools import lru_cache
from pathlib import Path
import joblib
import numpy as np
import pandas as pd
HERE = Path(__file__).resolve().parent
NUMERIC_COLS = [
'statuses_count', 'followers_count', 'following_count',
'username_length', 'username_num_digits',
'username_num_letters', 'username_num_special',
'username_starts_with_digit', 'username_ends_with_digit',
]
UNKNOWN_LANG = 'unknown'
OTHER_LANG = 'other'
MODEL_PATH = HERE / 'logistic_regression_model.joblib'
SCALER_PATH = HERE / 'scaler.joblib'
LANG_VOCAB_PATH = HERE / 'language_vocab.json'
# fastText lid.176 (CC-BY-SA 3.0, Facebook) is not redistributed here; it is fetched on first use.
FASTTEXT_URL = 'https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz'
FASTTEXT_MODEL_PATH = Path(os.getenv(
'FASTTEXT_MODEL_PATH',
Path.home() / '.cache' / 'fediverse-bot-detector' / 'lid.176.ftz',
))
def extract_username_features(username):
username = username or ''
return {
'username_length': len(username),
'username_num_digits': sum(c.isdigit() for c in username),
'username_num_letters': sum(c.isalpha() for c in username),
'username_num_special': sum(not c.isalnum() for c in username),
'username_starts_with_digit': int(bool(username) and username[0].isdigit()),
'username_ends_with_digit': int(bool(username) and username[-1].isdigit()),
}
def _ensure_fasttext_model():
if not FASTTEXT_MODEL_PATH.exists():
FASTTEXT_MODEL_PATH.parent.mkdir(parents=True, exist_ok=True)
tmp = FASTTEXT_MODEL_PATH.with_suffix('.part')
urllib.request.urlretrieve(FASTTEXT_URL, tmp)
tmp.replace(FASTTEXT_MODEL_PATH)
return FASTTEXT_MODEL_PATH
@lru_cache(maxsize=1)
def _fasttext_model():
import fasttext
return fasttext.load_model(str(_ensure_fasttext_model()))
def normalize_lang(lang):
if not isinstance(lang, str) or not lang.strip():
return UNKNOWN_LANG
return lang.replace('__label__', '').strip().lower()
def detect_language(text):
"""fastText lid.176 language code of a bio; 'unknown' when empty."""
if not isinstance(text, str) or not text.strip():
return UNKNOWN_LANG
# model.f.predict avoids the numpy-2 incompatibility of FastText.predict
preds = _fasttext_model().f.predict(text.replace('\n', ' '), 1, 0.0, 'strict')
return normalize_lang(preds[0][1]) if preds else UNKNOWN_LANG
def language_onehot(languages, vocab):
"""One-hot over vocab + OTHER_LANG. Languages outside vocab map to 'other'."""
columns = list(vocab) + [OTHER_LANG]
index = {lang: i for i, lang in enumerate(columns)}
out = np.zeros((len(languages), len(columns)), dtype=np.float32)
for row, lang in enumerate(languages):
out[row, index.get(normalize_lang(lang), index[OTHER_LANG])] = 1.0
return out
def add_account_features(df, text_col='clean_note'):
"""Add the username and language columns."""
df = df.copy()
missing = [c for c in NUMERIC_COLS[3:9] if c not in df.columns]
if missing:
username_features = df['username'].apply(extract_username_features).apply(pd.Series)
df = pd.concat([df.drop(columns=missing, errors='ignore'), username_features], axis=1)
if 'language' not in df.columns:
df['language'] = [detect_language(t) for t in df[text_col]]
return df
def load_artifacts():
with open(LANG_VOCAB_PATH) as f:
vocab = json.load(f)
return joblib.load(MODEL_PATH), joblib.load(SCALER_PATH), vocab
def build_feature_matrix(df, embeddings, scaler, vocab):
"""[scaled numeric | language one-hot | embeddings]; df must already have all features."""
numeric = scaler.transform(df[NUMERIC_COLS].astype(float).values)
return np.hstack((numeric, language_onehot(df['language'].tolist(), vocab), embeddings))