File size: 1,811 Bytes
46cc63a | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 | # Clean-Signal Dual-Input Hybrid — raw Text (BERT) + clean_text/metadata (LR)
pipeline:
random_state: 42
test_size: 0.2
val_size: 0.15
max_train_test_gap: 0.05
target_f1_weighted: 0.80
data:
raw_path: data/raw/youtoxic_english_1000.csv
processed_preprocessed: data/processed/v2/comments_preprocessed.csv
processed_stats: data/processed/v2/comments_with_stats.csv
target_binary: IsToxic
text_column: Text
id_column: CommentId
features_config: configs/features.yaml
augmentation:
enabled: true
strategy: back_translation
source_lang: en
pivot_lang: de
min_words: 3
max_words: 60
rate_limit_every: 50
rate_limit_sleep_sec: 1.0
dedup:
enabled: true
cosine_threshold: 0.95
embedding_model: sentence-transformers/all-MiniLM-L6-v2
transformer:
model_id: unitary/toxic-bert
max_length: 128
reuse_checkpoint: models/expert_toxic_bert
fixed_threshold: 0.33
train_if_missing: false
logistic_regression:
C: 0.05
max_iter: 2000
class_weight: balanced
solver: lbfgs
gap_search:
enabled: true
max_gap: 0.05
use_original_train_for_gap: true
param_grid:
- {C: 0.05, max_features: 800, min_df: 3}
- {C: 0.03, max_features: 500, min_df: 5}
- {C: 0.02, max_features: 400, min_df: 5}
- {C: 0.01, max_features: 400, min_df: 8}
- {C: 0.005, max_features: 300, min_df: 10}
- {C: 0.002, max_features: 250, min_df: 12}
tfidf:
max_features: 800
ngram_range: [1, 2]
sublinear_tf: true
min_df: 3
ensemble:
weight_metric: f1_weighted
min_lr_weight: 0.15
max_lr_weight: 0.45
threshold_tuning:
enabled: true
metric: f1_weighted
output:
lr_path: models/hybrid_clean_lr.joblib
ensemble_meta_path: models/hybrid_clean_ensemble_meta.json
reports_dir: reports/hybrid_clean
|