omnisentry-text / preprocessing_config.json
HBIQ's picture
Upload 12 files
9e0002d verified
raw
history blame contribute delete
576 Bytes
{
"preprocessing_params": {
"max_length": 512,
"mean_human_length": 1783.427587582612,
"mean_ai_length": 1798.8007539931875,
"total_samples": 487231,
"preprocessing_timestamp": "2025-08-09T03:53:51.664833"
},
"text_length_stats": {
"mean_length": 1789.2861020462615,
"max_length_used": 466,
"percentile_95": 466.69999999999993,
"percentile_99": 503.09000000000003
},
"dataset_stats": {
"total_samples": 487231,
"train_samples": 389784,
"test_samples": 97447,
"human_samples": 305797,
"ai_samples": 181434
}
}