| evaluations: |
| arc_challenge_poly_pt_acc: 0.564957264957265 |
| arc_challenge_poly_pt_acc_norm: 0.6034188034188034 |
| arc_challenge_poly_pt_acc_norm_stderr: 0.014307647225117459 |
| arc_challenge_poly_pt_acc_stderr: 0.014499949963905044 |
| arc_challenge_poly_pt_alias: arc_challenge_poly_pt |
| assin2_rte_acc,all: 0.9268790849673203 |
| assin2_rte_acc_stderr,all: 0.0037334279295552935 |
| assin2_rte_alias: assin2_rte |
| assin2_rte_f1_macro,all: 0.9275512463143912 |
| assin2_rte_f1_macro_stderr,all: 0.00371420765851098 |
| assin2_sts_alias: assin2_sts |
| assin2_sts_mse,all: 1.0666135620915032 |
| assin2_sts_mse_stderr,all: N/A |
| assin2_sts_pearson,all: 0.7025111393506918 |
| assin2_sts_pearson_stderr,all: 0.007270347714019143 |
| assin_entailment_acc: 0.729 |
| assin_entailment_acc_stderr: 0.00702866278356924 |
| assin_entailment_alias: assin_entailment |
| assin_paraphrase_acc: 0.70175 |
| assin_paraphrase_acc_stderr: 0.007234453587153363 |
| assin_paraphrase_alias: assin_paraphrase |
| belebele_por_Latn_acc: 0.8522222222222222 |
| belebele_por_Latn_acc_norm: 0.8522222222222222 |
| belebele_por_Latn_acc_norm_stderr: 0.011835896183094718 |
| belebele_por_Latn_acc_stderr: 0.011835896183094718 |
| belebele_por_Latn_alias: belebele_por_Latn |
| bluex_acc,all: 0.6453407510431154 |
| bluex_acc,exam_id__UNICAMP_2018: 0.5740740740740741 |
| bluex_acc,exam_id__UNICAMP_2019: 0.66 |
| bluex_acc,exam_id__UNICAMP_2020: 0.6545454545454545 |
| bluex_acc,exam_id__UNICAMP_2021_1: 0.5217391304347826 |
| bluex_acc,exam_id__UNICAMP_2021_2: 0.5686274509803921 |
| bluex_acc,exam_id__UNICAMP_2022: 0.7692307692307693 |
| bluex_acc,exam_id__UNICAMP_2023: 0.6976744186046512 |
| bluex_acc,exam_id__UNICAMP_2024: 0.6444444444444445 |
| bluex_acc,exam_id__USP_2018: 0.5555555555555556 |
| bluex_acc,exam_id__USP_2019: 0.6 |
| bluex_acc,exam_id__USP_2020: 0.6428571428571429 |
| bluex_acc,exam_id__USP_2021: 0.75 |
| bluex_acc,exam_id__USP_2022: 0.6122448979591837 |
| bluex_acc,exam_id__USP_2023: 0.7272727272727273 |
| bluex_acc,exam_id__USP_2024: 0.7560975609756098 |
| bluex_acc_stderr,all: 0.01030518267625725 |
| bluex_acc_stderr,exam_id__UNICAMP_2018: 0.03890130999517633 |
| bluex_acc_stderr,exam_id__UNICAMP_2019: 0.038803737151553275 |
| bluex_acc_stderr,exam_id__UNICAMP_2020: 0.03685887038229413 |
| bluex_acc_stderr,exam_id__UNICAMP_2021_1: 0.04271355498682148 |
| bluex_acc_stderr,exam_id__UNICAMP_2021_2: 0.0399126533750554 |
| bluex_acc_stderr,exam_id__UNICAMP_2022: 0.03889086167730691 |
| bluex_acc_stderr,exam_id__UNICAMP_2023: 0.040270225038111215 |
| bluex_acc_stderr,exam_id__UNICAMP_2024: 0.04120765210662655 |
| bluex_acc_stderr,exam_id__USP_2018: 0.039103130248058574 |
| bluex_acc_stderr,exam_id__USP_2019: 0.04468770985550053 |
| bluex_acc_stderr,exam_id__USP_2020: 0.036928680498947423 |
| bluex_acc_stderr,exam_id__USP_2021: 0.03469119064952554 |
| bluex_acc_stderr,exam_id__USP_2022: 0.0401889704113241 |
| bluex_acc_stderr,exam_id__USP_2023: 0.03875202753310343 |
| bluex_acc_stderr,exam_id__USP_2024: 0.0386460634988709 |
| bluex_alias: bluex |
| calame_pt_acc: 0.49710982658959535 |
| calame_pt_acc_stderr: 0.010976242623017822 |
| calame_pt_alias: calame_pt |
| calame_pt_perplexity: 9.89153235647868 |
| calame_pt_perplexity_stderr: 0.5360166925656309 |
| enem_challenge_acc,all: 0.7291812456263121 |
| enem_challenge_acc,exam_id__2009: 0.6695652173913044 |
| enem_challenge_acc,exam_id__2010: 0.7948717948717948 |
| enem_challenge_acc,exam_id__2011: 0.8034188034188035 |
| enem_challenge_acc,exam_id__2012: 0.7672413793103449 |
| enem_challenge_acc,exam_id__2013: 0.7685185185185185 |
| enem_challenge_acc,exam_id__2014: 0.7339449541284404 |
| enem_challenge_acc,exam_id__2015: 0.7142857142857143 |
| enem_challenge_acc,exam_id__2016: 0.7024793388429752 |
| enem_challenge_acc,exam_id__2016_2: 0.6910569105691057 |
| enem_challenge_acc,exam_id__2017: 0.6810344827586207 |
| enem_challenge_acc,exam_id__2022: 0.6842105263157895 |
| enem_challenge_acc,exam_id__2023: 0.7481481481481481 |
| enem_challenge_acc_stderr,all: 0.006807641973418005 |
| enem_challenge_acc_stderr,exam_id__2009: 0.02528804994180274 |
| enem_challenge_acc_stderr,exam_id__2010: 0.02149640668210639 |
| enem_challenge_acc_stderr,exam_id__2011: 0.021210692080503628 |
| enem_challenge_acc_stderr,exam_id__2012: 0.022660217278927836 |
| enem_challenge_acc_stderr,exam_id__2013: 0.02336336231167537 |
| enem_challenge_acc_stderr,exam_id__2014: 0.024490632130058823 |
| enem_challenge_acc_stderr,exam_id__2015: 0.023816151603437377 |
| enem_challenge_acc_stderr,exam_id__2016: 0.023907462731025793 |
| enem_challenge_acc_stderr,exam_id__2016_2: 0.024064368893731206 |
| enem_challenge_acc_stderr,exam_id__2017: 0.024896265014785066 |
| enem_challenge_acc_stderr,exam_id__2022: 0.023240775852571182 |
| enem_challenge_acc_stderr,exam_id__2023: 0.021584384605568642 |
| enem_challenge_alias: enem |
| faquad_nli_acc,all: 0.8584615384615385 |
| faquad_nli_acc_stderr,all: 0.00964563979812997 |
| faquad_nli_alias: faquad_nli |
| faquad_nli_f1_macro,all: 0.7500687682235792 |
| faquad_nli_f1_macro_stderr,all: 0.016061769703906006 |
| global_piqa_completions_por_latn_braz_acc: 0.84 |
| global_piqa_completions_por_latn_braz_acc_bytes: 0.78 |
| global_piqa_completions_por_latn_braz_acc_bytes_stderr: 0.041633319989322654 |
| global_piqa_completions_por_latn_braz_acc_norm: 0.8 |
| global_piqa_completions_por_latn_braz_acc_norm_stderr: 0.04020151261036849 |
| global_piqa_completions_por_latn_braz_acc_stderr: 0.03684529491774706 |
| global_piqa_completions_por_latn_braz_alias: global_piqa_completions_por_latn_braz |
| gsm8k_pt_alias: gsm8k_pt |
| gsm8k_pt_exact_match,flexible-extract: 0.5380517503805176 |
| gsm8k_pt_exact_match,strict-match: 0.0 |
| gsm8k_pt_exact_match_stderr,flexible-extract: 0.013758665878004294 |
| gsm8k_pt_exact_match_stderr,strict-match: 0.0 |
| hatebr_offensive_acc,all: 0.8035714285714286 |
| hatebr_offensive_acc_stderr,all: 0.007485688659514488 |
| hatebr_offensive_alias: hatebr_offensive_binary |
| hatebr_offensive_f1_macro,all: 0.7971547930214926 |
| hatebr_offensive_f1_macro_stderr,all: 0.0077008767186672504 |
| hellaswag_poly_pt_acc: 0.4855347274894355 |
| hellaswag_poly_pt_acc_norm: 0.6400476758045291 |
| hellaswag_poly_pt_acc_norm_stderr: 0.0049966031862365996 |
| hellaswag_poly_pt_acc_stderr: 0.00520276713464127 |
| hellaswag_poly_pt_alias: hellaswag_poly_pt |
| humaneval_instruct_alias: humaneval_instruct |
| humaneval_instruct_pass@1,create_test: 0.47560975609756095 |
| humaneval_instruct_pass@1_stderr,create_test: 0.039116399837036665 |
| ifeval_pt_alias: ifeval_pt |
| ifeval_pt_inst_level_loose_acc: 0.5209302325581395 |
| ifeval_pt_inst_level_loose_acc_stderr: N/A |
| ifeval_pt_inst_level_strict_acc: 0.48604651162790696 |
| ifeval_pt_inst_level_strict_acc_stderr: N/A |
| ifeval_pt_prompt_level_loose_acc: 0.4166666666666667 |
| ifeval_pt_prompt_level_loose_acc_stderr: 0.028511310643917525 |
| ifeval_pt_prompt_level_strict_acc: 0.38666666666666666 |
| ifeval_pt_prompt_level_strict_acc_stderr: 0.028163138908196883 |
| lambada_poly_pt_acc: 0.5893654182029886 |
| lambada_poly_pt_acc_stderr: 0.006853811533501879 |
| lambada_poly_pt_alias: lambada_poly_pt |
| lambada_poly_pt_perplexity: 10.134180916859764 |
| lambada_poly_pt_perplexity_stderr: 0.3843105646542514 |
| mmlu_poly_pt_acc: 0.6463524467126989 |
| mmlu_poly_pt_acc_stderr: 0.004142085063565535 |
| mmlu_poly_pt_alias: mmlu_poly_pt |
| oab_exams_acc,all: 0.5430523917995445 |
| oab_exams_acc,exam_id__2010-01: 0.38823529411764707 |
| oab_exams_acc,exam_id__2010-02: 0.52 |
| oab_exams_acc,exam_id__2011-03: 0.48484848484848486 |
| oab_exams_acc,exam_id__2011-04: 0.5 |
| oab_exams_acc,exam_id__2011-05: 0.6125 |
| oab_exams_acc,exam_id__2012-06: 0.55 |
| oab_exams_acc,exam_id__2012-06a: 0.6625 |
| oab_exams_acc,exam_id__2012-07: 0.55 |
| oab_exams_acc,exam_id__2012-08: 0.525 |
| oab_exams_acc,exam_id__2012-09: 0.4675324675324675 |
| oab_exams_acc,exam_id__2013-10: 0.525 |
| oab_exams_acc,exam_id__2013-11: 0.525 |
| oab_exams_acc,exam_id__2013-12: 0.5625 |
| oab_exams_acc,exam_id__2014-13: 0.5375 |
| oab_exams_acc,exam_id__2014-14: 0.525 |
| oab_exams_acc,exam_id__2014-15: 0.6282051282051282 |
| oab_exams_acc,exam_id__2015-16: 0.5 |
| oab_exams_acc,exam_id__2015-17: 0.6282051282051282 |
| oab_exams_acc,exam_id__2015-18: 0.5625 |
| oab_exams_acc,exam_id__2016-19: 0.6153846153846154 |
| oab_exams_acc,exam_id__2016-20: 0.6625 |
| oab_exams_acc,exam_id__2016-20a: 0.5125 |
| oab_exams_acc,exam_id__2016-21: 0.55 |
| oab_exams_acc,exam_id__2017-22: 0.5375 |
| oab_exams_acc,exam_id__2017-23: 0.5625 |
| oab_exams_acc,exam_id__2017-24: 0.5625 |
| oab_exams_acc,exam_id__2018-25: 0.4375 |
| oab_exams_acc_stderr,all: 0.006135349097679556 |
| oab_exams_acc_stderr,exam_id__2010-01: 0.03058058457150523 |
| oab_exams_acc_stderr,exam_id__2010-02: 0.02878728598780894 |
| oab_exams_acc_stderr,exam_id__2011-03: 0.028914895804693044 |
| oab_exams_acc_stderr,exam_id__2011-04: 0.03228642008607808 |
| oab_exams_acc_stderr,exam_id__2011-05: 0.031320378814850496 |
| oab_exams_acc_stderr,exam_id__2012-06: 0.0321418972062254 |
| oab_exams_acc_stderr,exam_id__2012-06a: 0.030467398275719 |
| oab_exams_acc_stderr,exam_id__2012-07: 0.03200736259141147 |
| oab_exams_acc_stderr,exam_id__2012-08: 0.032266439816730046 |
| oab_exams_acc_stderr,exam_id__2012-09: 0.03267111052784679 |
| oab_exams_acc_stderr,exam_id__2013-10: 0.03227871954884383 |
| oab_exams_acc_stderr,exam_id__2013-11: 0.032317288738860646 |
| oab_exams_acc_stderr,exam_id__2013-12: 0.03201360716056409 |
| oab_exams_acc_stderr,exam_id__2014-13: 0.03212888100414898 |
| oab_exams_acc_stderr,exam_id__2014-14: 0.03224480657768177 |
| oab_exams_acc_stderr,exam_id__2014-15: 0.03161561320473922 |
| oab_exams_acc_stderr,exam_id__2015-16: 0.032285769779197525 |
| oab_exams_acc_stderr,exam_id__2015-17: 0.03149277459357065 |
| oab_exams_acc_stderr,exam_id__2015-18: 0.03193661718329485 |
| oab_exams_acc_stderr,exam_id__2016-19: 0.031871219023965544 |
| oab_exams_acc_stderr,exam_id__2016-20: 0.030531876914015273 |
| oab_exams_acc_stderr,exam_id__2016-20a: 0.03228630691735584 |
| oab_exams_acc_stderr,exam_id__2016-21: 0.03207739680674974 |
| oab_exams_acc_stderr,exam_id__2017-22: 0.03209350013865321 |
| oab_exams_acc_stderr,exam_id__2017-23: 0.03217100216372156 |
| oab_exams_acc_stderr,exam_id__2017-24: 0.032070126882455134 |
| oab_exams_acc_stderr,exam_id__2018-25: 0.03198404003914565 |
| oab_exams_alias: oab_exams |
| portuguese_hate_speech_acc,all: 0.6698002350176263 |
| portuguese_hate_speech_acc_stderr,all: 0.011429241144055021 |
| portuguese_hate_speech_alias: portuguese_hate_speech_binary |
| portuguese_hate_speech_f1_macro,all: 0.6614050364958336 |
| portuguese_hate_speech_f1_macro_stderr,all: 0.01163030492382994 |
| tweetsentbr_acc,all: 0.7069651741293532 |
| tweetsentbr_acc_stderr,all: 0.007157154484263642 |
| tweetsentbr_alias: tweetsentbr |
| tweetsentbr_f1_macro,all: 0.6707907518730517 |
| tweetsentbr_f1_macro_stderr,all: 0.007569663916929406 |
| step: 69750 |
|
|
