| --- |
| library_name: transformers |
| tags: [] |
| --- |
| # DACTYL Text Detector |
| ## Training Configuration |
| ```json |
| { |
| "training_split": "training", |
| "evaluation_split": "testing", |
| "results_path": "libauc-bert-tiny.csv", |
| "num_epochs": 1, |
| "model_path": "ShantanuT01/dactyl-bert-tiny-pretrained", |
| "tokenizer": "prajjwal1/bert-tiny", |
| "optimizer": "SOTAs", |
| "optimizer_type": "libauc", |
| "optimizer_args": { |
| "lr": 1e-05 |
| }, |
| "loss_fn": "tpAUC_KL_Loss", |
| "reset_classification_head": true, |
| "loss_type": "libauc", |
| "loss_fn_args": { |
| "data_len": 466005 |
| }, |
| "needs_loss_fn_as_parameter": false, |
| "save_path": "ShantanuT01/dactyl-bert-tiny-finetuned", |
| "training_args": { |
| "batch_size": 64, |
| "needs_sampler": true, |
| "needs_index": true, |
| "shuffle": false, |
| "sampling_rate": 0.5, |
| "apply_sigmoid": true |
| }, |
| "best_model_path": "best-tpauc-model-bert-tiny" |
| } |
| ``` |
| ## Results |
| | model | AP Score | AUC Score | OPAUC Score | TPAUC Score | |
| |:--------------------------------------------------------------------------------------|-----------:|------------:|--------------:|--------------:| |
| | DeepSeek-V3 | 0.986177 | 0.997357 | 0.991422 | 0.916469 | |
| | ShantanuT01/fine-tuned-Llama-3.2-1B-Instruct-apollo-mini-RedditWritingPrompts-testing | 0.0242264 | 0.805738 | 0.619715 | 0 | |
| | ShantanuT01/fine-tuned-Llama-3.2-1B-Instruct-apollo-mini-abstracts-testing | 0.473122 | 0.955828 | 0.857774 | 0.0525074 | |
| | ShantanuT01/fine-tuned-Llama-3.2-1B-Instruct-apollo-mini-news-testing | 0.0223995 | 0.776301 | 0.579092 | 0 | |
| | ShantanuT01/fine-tuned-Llama-3.2-1B-Instruct-apollo-mini-reviews-testing | 0.0167409 | 0.861497 | 0.526359 | 0 | |
| | ShantanuT01/fine-tuned-Llama-3.2-1B-Instruct-apollo-mini-student_essays-testing | 0.0102312 | 0.621629 | 0.51884 | 0 | |
| | ShantanuT01/fine-tuned-Llama-3.2-1B-Instruct-apollo-mini-tweets-testing | 0.200169 | 0.949163 | 0.728122 | 0 | |
| | claude-3-5-haiku-20241022 | 0.96485 | 0.993769 | 0.977445 | 0.780686 | |
| | claude-3-5-sonnet-20241022 | 0.981121 | 0.997236 | 0.987999 | 0.883237 | |
| | gemini-1.5-flash | 0.93374 | 0.989888 | 0.955699 | 0.57296 | |
| | gemini-1.5-pro | 0.856575 | 0.96967 | 0.909663 | 0.190672 | |
| | gpt-4o-2024-11-20 | 0.950062 | 0.990051 | 0.968528 | 0.695722 | |
| | gpt-4o-mini | 0.992634 | 0.999149 | 0.995225 | 0.953515 | |
| | llama-3.2-90b | 0.895571 | 0.978201 | 0.933229 | 0.366601 | |
| | llama-3.3-70b | 0.952439 | 0.992252 | 0.968929 | 0.699379 | |
| | mistral-large-latest | 0.979345 | 0.996937 | 0.986758 | 0.871375 | |
| | mistral-small-latest | 0.986836 | 0.997499 | 0.991305 | 0.915284 | |
| | overall | 0.976784 | 0.980759 | 0.947371 | 0.492028 | |
| |
