FranUPM's picture
Initial release: logistic regression fediverse bot detector
8793683 verified
Raw History Blame Contribute Delete
3.38 kB
"""Inference for the fediverse bot classifier.
from huggingface_hub import snapshot_download
import sys
path = snapshot_download("FranUPM/fediverse-bot-detector")
sys.path.insert(0, path)
from predict import BotClassifier
clf = BotClassifier()
clf.predict([{"username": "news_bot_2024", "note": "<p>Automated headlines every hour</p>",
"statuses_count": 52000, "followers_count": 120, "following_count": 0}])
"""
import numpy as np
import pandas as pd
import torch
from bs4 import BeautifulSoup
from transformers import BertModel, BertTokenizer
from bot_features import add_account_features, build_feature_matrix, load_artifacts
EMBEDDING_MODEL = 'bert-base-multilingual-cased'
REQUIRED_FIELDS = ('username', 'note', 'statuses_count', 'followers_count', 'following_count')
def remove_html(text):
if not isinstance(text, str):
return ''
return BeautifulSoup(text, 'html.parser').get_text(separator=' ', strip=True)
class BotClassifier:
def __init__(self, device=None, threshold=0.5):
self.model, self.scaler, self.vocab = load_artifacts()
self.device = torch.device(device or ('cuda' if torch.cuda.is_available() else 'cpu'))
self.tokenizer = BertTokenizer.from_pretrained(EMBEDDING_MODEL)
self.bert = BertModel.from_pretrained(EMBEDDING_MODEL).to(self.device).eval()
self.threshold = threshold
@torch.no_grad()
def embed(self, texts, max_length=128, batch_size=32):
"""[CLS] embedding of each text (768-dim), same as used in training."""
out = []
for i in range(0, len(texts), batch_size):
inputs = self.tokenizer(texts[i:i + batch_size], return_tensors='pt',
truncation=True, padding=True, max_length=max_length)
inputs = {k: v.to(self.device) for k, v in inputs.items()}
out.append(self.bert(**inputs).last_hidden_state[:, 0, :].cpu().numpy())
return np.vstack(out)
def predict(self, accounts):
"""accounts: list of dicts with username, note (HTML bio), statuses_count,
followers_count, following_count. Optional 'language' skips fastText detection."""
if isinstance(accounts, dict):
accounts = [accounts]
for a in accounts:
missing = [f for f in REQUIRED_FIELDS if f not in a]
if missing:
raise ValueError(f"account {a.get('username')!r} is missing {missing}")
df = pd.DataFrame(accounts)
df['clean_note'] = df['note'].apply(remove_html)
df = add_account_features(df)
emb = self.embed(df['clean_note'].fillna('').astype(str).tolist())
X = build_feature_matrix(df, emb, self.scaler, self.vocab)
proba = self.model.predict_proba(X)[:, 1]
return [{'username': u, 'prediction': bool(p >= self.threshold), 'probability': float(p)}
for u, p in zip(df['username'], proba)]
if __name__ == '__main__':
clf = BotClassifier()
print(clf.predict([
{'username': 'news_bot_2024', 'note': '<p>Automated headlines posted every hour.</p>',
'statuses_count': 52000, 'followers_count': 120, 'following_count': 0},
{'username': 'maria', 'note': '<p>Biologist, cyclist, coffee lover.</p>',
'statuses_count': 830, 'followers_count': 310, 'following_count': 280},
]))