Tabular Classification
Scikit-learn
Joblib
multilingual
bot-detection
mastodon
fediverse
logistic-regression
Instructions to use FranUPM/fediverse-bot-detector with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Scikit-learn
How to use FranUPM/fediverse-bot-detector with Scikit-learn:
from huggingface_hub import hf_hub_download import joblib model = joblib.load( hf_hub_download("FranUPM/fediverse-bot-detector", "sklearn_model.joblib") ) # only load pickle files from sources you trust # read more about it here https://skops.readthedocs.io/en/stable/persistence.html - Notebooks
- Google Colab
- Kaggle
Download predict.py from FranUPM/fediverse-bot-detector: direct link, hf CLI and curl.
- Browser
- Download file 3.38 kB
-
https://huggingface.co/FranUPM/fediverse-bot-detector/resolve/main/predict.py
- Command line
-
hf download hf://FranUPM/fediverse-bot-detector/predict.py
-
curl -L -o predict.py https://huggingface.co/FranUPM/fediverse-bot-detector/resolve/main/predict.py
3.38 kB
| """Inference for the fediverse bot classifier. | |
| from huggingface_hub import snapshot_download | |
| import sys | |
| path = snapshot_download("FranUPM/fediverse-bot-detector") | |
| sys.path.insert(0, path) | |
| from predict import BotClassifier | |
| clf = BotClassifier() | |
| clf.predict([{"username": "news_bot_2024", "note": "<p>Automated headlines every hour</p>", | |
| "statuses_count": 52000, "followers_count": 120, "following_count": 0}]) | |
| """ | |
| import numpy as np | |
| import pandas as pd | |
| import torch | |
| from bs4 import BeautifulSoup | |
| from transformers import BertModel, BertTokenizer | |
| from bot_features import add_account_features, build_feature_matrix, load_artifacts | |
| EMBEDDING_MODEL = 'bert-base-multilingual-cased' | |
| REQUIRED_FIELDS = ('username', 'note', 'statuses_count', 'followers_count', 'following_count') | |
| def remove_html(text): | |
| if not isinstance(text, str): | |
| return '' | |
| return BeautifulSoup(text, 'html.parser').get_text(separator=' ', strip=True) | |
| class BotClassifier: | |
| def __init__(self, device=None, threshold=0.5): | |
| self.model, self.scaler, self.vocab = load_artifacts() | |
| self.device = torch.device(device or ('cuda' if torch.cuda.is_available() else 'cpu')) | |
| self.tokenizer = BertTokenizer.from_pretrained(EMBEDDING_MODEL) | |
| self.bert = BertModel.from_pretrained(EMBEDDING_MODEL).to(self.device).eval() | |
| self.threshold = threshold | |
| def embed(self, texts, max_length=128, batch_size=32): | |
| """[CLS] embedding of each text (768-dim), same as used in training.""" | |
| out = [] | |
| for i in range(0, len(texts), batch_size): | |
| inputs = self.tokenizer(texts[i:i + batch_size], return_tensors='pt', | |
| truncation=True, padding=True, max_length=max_length) | |
| inputs = {k: v.to(self.device) for k, v in inputs.items()} | |
| out.append(self.bert(**inputs).last_hidden_state[:, 0, :].cpu().numpy()) | |
| return np.vstack(out) | |
| def predict(self, accounts): | |
| """accounts: list of dicts with username, note (HTML bio), statuses_count, | |
| followers_count, following_count. Optional 'language' skips fastText detection.""" | |
| if isinstance(accounts, dict): | |
| accounts = [accounts] | |
| for a in accounts: | |
| missing = [f for f in REQUIRED_FIELDS if f not in a] | |
| if missing: | |
| raise ValueError(f"account {a.get('username')!r} is missing {missing}") | |
| df = pd.DataFrame(accounts) | |
| df['clean_note'] = df['note'].apply(remove_html) | |
| df = add_account_features(df) | |
| emb = self.embed(df['clean_note'].fillna('').astype(str).tolist()) | |
| X = build_feature_matrix(df, emb, self.scaler, self.vocab) | |
| proba = self.model.predict_proba(X)[:, 1] | |
| return [{'username': u, 'prediction': bool(p >= self.threshold), 'probability': float(p)} | |
| for u, p in zip(df['username'], proba)] | |
| if __name__ == '__main__': | |
| clf = BotClassifier() | |
| print(clf.predict([ | |
| {'username': 'news_bot_2024', 'note': '<p>Automated headlines posted every hour.</p>', | |
| 'statuses_count': 52000, 'followers_count': 120, 'following_count': 0}, | |
| {'username': 'maria', 'note': '<p>Biologist, cyclist, coffee lover.</p>', | |
| 'statuses_count': 830, 'followers_count': 310, 'following_count': 280}, | |
| ])) | |