Tabular Classification
Scikit-learn
Joblib
multilingual
bot-detection
mastodon
fediverse
logistic-regression
Instructions to use FranUPM/fediverse-bot-detector with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Scikit-learn
How to use FranUPM/fediverse-bot-detector with Scikit-learn:
from huggingface_hub import hf_hub_download import joblib model = joblib.load( hf_hub_download("FranUPM/fediverse-bot-detector", "sklearn_model.joblib") ) # only load pickle files from sources you trust # read more about it here https://skops.readthedocs.io/en/stable/persistence.html - Notebooks
- Google Colab
- Kaggle
Download bot_features.py from FranUPM/fediverse-bot-detector: direct link, hf CLI and curl.
- Browser
- Download file 4.13 kB
-
https://huggingface.co/FranUPM/fediverse-bot-detector/resolve/main/bot_features.py
- Command line
-
hf download hf://FranUPM/fediverse-bot-detector/bot_features.py
-
curl -L -o bot_features.py https://huggingface.co/FranUPM/fediverse-bot-detector/resolve/main/bot_features.py
4.13 kB
| """Feature engineering for the fediverse bot classifier. | |
| Feature layout fed to the model: [scaled numeric | language one-hot | bio embedding]. | |
| fastText is imported lazily and its language-ID model is downloaded on first use. | |
| """ | |
| import json | |
| import os | |
| import urllib.request | |
| from functools import lru_cache | |
| from pathlib import Path | |
| import joblib | |
| import numpy as np | |
| import pandas as pd | |
| HERE = Path(__file__).resolve().parent | |
| NUMERIC_COLS = [ | |
| 'statuses_count', 'followers_count', 'following_count', | |
| 'username_length', 'username_num_digits', | |
| 'username_num_letters', 'username_num_special', | |
| 'username_starts_with_digit', 'username_ends_with_digit', | |
| ] | |
| UNKNOWN_LANG = 'unknown' | |
| OTHER_LANG = 'other' | |
| MODEL_PATH = HERE / 'logistic_regression_model.joblib' | |
| SCALER_PATH = HERE / 'scaler.joblib' | |
| LANG_VOCAB_PATH = HERE / 'language_vocab.json' | |
| # fastText lid.176 (CC-BY-SA 3.0, Facebook) is not redistributed here; it is fetched on first use. | |
| FASTTEXT_URL = 'https://dl.fbaipublicfiles.com/fasttext/supervised-models/lid.176.ftz' | |
| FASTTEXT_MODEL_PATH = Path(os.getenv( | |
| 'FASTTEXT_MODEL_PATH', | |
| Path.home() / '.cache' / 'fediverse-bot-detector' / 'lid.176.ftz', | |
| )) | |
| def extract_username_features(username): | |
| username = username or '' | |
| return { | |
| 'username_length': len(username), | |
| 'username_num_digits': sum(c.isdigit() for c in username), | |
| 'username_num_letters': sum(c.isalpha() for c in username), | |
| 'username_num_special': sum(not c.isalnum() for c in username), | |
| 'username_starts_with_digit': int(bool(username) and username[0].isdigit()), | |
| 'username_ends_with_digit': int(bool(username) and username[-1].isdigit()), | |
| } | |
| def _ensure_fasttext_model(): | |
| if not FASTTEXT_MODEL_PATH.exists(): | |
| FASTTEXT_MODEL_PATH.parent.mkdir(parents=True, exist_ok=True) | |
| tmp = FASTTEXT_MODEL_PATH.with_suffix('.part') | |
| urllib.request.urlretrieve(FASTTEXT_URL, tmp) | |
| tmp.replace(FASTTEXT_MODEL_PATH) | |
| return FASTTEXT_MODEL_PATH | |
| def _fasttext_model(): | |
| import fasttext | |
| return fasttext.load_model(str(_ensure_fasttext_model())) | |
| def normalize_lang(lang): | |
| if not isinstance(lang, str) or not lang.strip(): | |
| return UNKNOWN_LANG | |
| return lang.replace('__label__', '').strip().lower() | |
| def detect_language(text): | |
| """fastText lid.176 language code of a bio; 'unknown' when empty.""" | |
| if not isinstance(text, str) or not text.strip(): | |
| return UNKNOWN_LANG | |
| # model.f.predict avoids the numpy-2 incompatibility of FastText.predict | |
| preds = _fasttext_model().f.predict(text.replace('\n', ' '), 1, 0.0, 'strict') | |
| return normalize_lang(preds[0][1]) if preds else UNKNOWN_LANG | |
| def language_onehot(languages, vocab): | |
| """One-hot over vocab + OTHER_LANG. Languages outside vocab map to 'other'.""" | |
| columns = list(vocab) + [OTHER_LANG] | |
| index = {lang: i for i, lang in enumerate(columns)} | |
| out = np.zeros((len(languages), len(columns)), dtype=np.float32) | |
| for row, lang in enumerate(languages): | |
| out[row, index.get(normalize_lang(lang), index[OTHER_LANG])] = 1.0 | |
| return out | |
| def add_account_features(df, text_col='clean_note'): | |
| """Add the username and language columns.""" | |
| df = df.copy() | |
| missing = [c for c in NUMERIC_COLS[3:9] if c not in df.columns] | |
| if missing: | |
| username_features = df['username'].apply(extract_username_features).apply(pd.Series) | |
| df = pd.concat([df.drop(columns=missing, errors='ignore'), username_features], axis=1) | |
| if 'language' not in df.columns: | |
| df['language'] = [detect_language(t) for t in df[text_col]] | |
| return df | |
| def load_artifacts(): | |
| with open(LANG_VOCAB_PATH) as f: | |
| vocab = json.load(f) | |
| return joblib.load(MODEL_PATH), joblib.load(SCALER_PATH), vocab | |
| def build_feature_matrix(df, embeddings, scaler, vocab): | |
| """[scaled numeric | language one-hot | embeddings]; df must already have all features.""" | |
| numeric = scaler.transform(df[NUMERIC_COLS].astype(float).values) | |
| return np.hstack((numeric, language_onehot(df['language'].tolist(), vocab), embeddings)) | |