| { |
| "config_general": { |
| "lighteval_sha": "?", |
| "num_fewshot_seeds": 1, |
| "override_batch_size": 8, |
| "max_samples": null, |
| "job_id": "", |
| "start_time": 2080.341543016, |
| "end_time": 2406.673661648, |
| "total_evaluation_time_secondes": "326.33211863199995", |
| "model_name": "tinycompany/BiBo-Mini-v0.9x", |
| "model_sha": "dec14e489d8cca7bc2df3cf4d70db4c5fe379279", |
| "model_dtype": "torch.bfloat16", |
| "model_size": "3.31 GB", |
| "config": null |
| }, |
| "results": { |
| "indiceval|MMLU:hindi|5": { |
| "acc": 0.3631249109813417, |
| "acc_stderr": 0.00405841199574701 |
| }, |
| "all": { |
| "acc": 0.3631249109813417, |
| "acc_stderr": 0.00405841199574701 |
| } |
| }, |
| "versions": { |
| "indiceval|MMLU:hindi|5": 0 |
| }, |
| "config_tasks": { |
| "indiceval|MMLU:hindi": { |
| "name": "MMLU:hindi", |
| "prompt_function": "mmlu_helm_indic", |
| "hf_repo": "Cognitive-Lab/Indic-MMLU", |
| "hf_subset": "hi", |
| "metric": [ |
| "loglikelihood_acc_single_token" |
| ], |
| "hf_avail_splits": [ |
| "test", |
| "validation", |
| "dev" |
| ], |
| "evaluation_splits": [ |
| "test" |
| ], |
| "few_shots_split": "dev", |
| "few_shots_select": "sequential", |
| "generation_size": 5, |
| "stop_sequence": [ |
| "\n" |
| ], |
| "output_regex": null, |
| "frozen": false, |
| "suite": [ |
| "indiceval", |
| "leaderboard", |
| "mmlu" |
| ], |
| "original_num_docs": 14042, |
| "effective_num_docs": 14042, |
| "trust_dataset": true, |
| "must_remove_duplicate_docs": null |
| } |
| }, |
| "summary_tasks": { |
| "indiceval|MMLU:hindi|5": { |
| "hashes": { |
| "hash_examples": "86da320415d99e54", |
| "hash_full_prompts": "7ba2716f08d15db0", |
| "hash_input_tokens": "5fa5c9f93a3d76c2", |
| "hash_cont_tokens": "f5200c7e7cd90dda" |
| }, |
| "truncated": 0, |
| "non_truncated": 14042, |
| "padded": 13793, |
| "non_padded": 249, |
| "effective_few_shots": 5.0, |
| "num_truncated_few_shots": 0 |
| } |
| }, |
| "summary_general": { |
| "hashes": { |
| "hash_examples": "9772337600b1daad", |
| "hash_full_prompts": "b11cd5ef781ad9af", |
| "hash_input_tokens": "22684734a1248a23", |
| "hash_cont_tokens": "c4caaf2e5148f67f" |
| }, |
| "truncated": 0, |
| "non_truncated": 14042, |
| "padded": 13793, |
| "non_padded": 249, |
| "num_truncated_few_shots": 0 |
| }, |
| "email": "fhai50032@gmail.com", |
| "language": "hindi" |
| } |
