| { |
| "results": { |
| "e5_frozen_logreg": { |
| "cv_accuracy": 0.8027444253859348, |
| "threshold_selection": { |
| "score": 0.7308870374908111, |
| "accepted_correct": 0.6046312178387651, |
| "rejected_ood": 0.8571428571428571 |
| }, |
| "threshold": 0.731022679863181, |
| "test": { |
| "overall": { |
| "value": 0.6949685534591195, |
| "ci95": [ |
| 0.6422860107063452, |
| 0.7429968563022046 |
| ], |
| "n": 318, |
| "successes": 221 |
| }, |
| "in_domain_correct": { |
| "value": 0.6949685534591195, |
| "ci95": [ |
| 0.6422860107063452, |
| 0.7429968563022046 |
| ], |
| "n": 318, |
| "successes": 221 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.8805031446540881, |
| "ci95": [ |
| 0.8402335419115275, |
| 0.9116894727341905 |
| ], |
| "n": 318, |
| "successes": 280 |
| }, |
| "by_group": { |
| "long": { |
| "value": 0.6981132075471698, |
| "ci95": [ |
| 0.5646316722203186, |
| 0.8048169734215012 |
| ], |
| "n": 53, |
| "successes": 37 |
| }, |
| "mixed": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6456695613241317, |
| 1.0 |
| ], |
| "n": 7, |
| "successes": 7 |
| }, |
| "plain": { |
| "value": 0.8080808080808081, |
| "ci95": [ |
| 0.7195963930846633, |
| 0.8735496072108871 |
| ], |
| "n": 99, |
| "successes": 80 |
| }, |
| "slang": { |
| "value": 0.6415094339622641, |
| "ci95": [ |
| 0.5069114144879365, |
| 0.7569804752562204 |
| ], |
| "n": 53, |
| "successes": 34 |
| }, |
| "translit": { |
| "value": 0.32075471698113206, |
| "ci95": [ |
| 0.2109221400637985, |
| 0.4548147995936025 |
| ], |
| "n": 53, |
| "successes": 17 |
| }, |
| "typo": { |
| "value": 0.8679245283018868, |
| "ci95": [ |
| 0.751597445822794, |
| 0.9345214675120143 |
| ], |
| "n": 53, |
| "successes": 46 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.8679245283018868, |
| "ci95": [ |
| 0.751597445822794, |
| 0.9345214675120143 |
| ], |
| "n": 53, |
| "successes": 46 |
| }, |
| "kk": { |
| "value": 0.7735849056603774, |
| "ci95": [ |
| 0.644737296378973, |
| 0.8654536904597305 |
| ], |
| "n": 53, |
| "successes": 41 |
| }, |
| "ru": { |
| "value": 0.6320754716981132, |
| "ci95": [ |
| 0.5653473327892489, |
| 0.6941023590829587 |
| ], |
| "n": 212, |
| "successes": 134 |
| } |
| }, |
| "macro_f1_no_threshold": 0.8839966637136447, |
| "per_intent": { |
| "about_university": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "academic_calendar": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "academic_integrity": { |
| "precision": 0.5, |
| "recall": 0.8333333333333334, |
| "f1": 0.625, |
| "support": 6 |
| }, |
| "academic_leave": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "academic_mobility": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "admission_deadlines": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "admission_documents": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "support": 6 |
| }, |
| "admission_steps": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "appeal": { |
| "precision": 0.625, |
| "recall": 0.8333333333333334, |
| "f1": 0.7142857142857143, |
| "support": 6 |
| }, |
| "capabilities": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "career_center": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "certificates": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "college_graduates": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "contacts": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "creative_exam": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "dormitory": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "ent_discounts": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "ent_subjects": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "faculties": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "foreign_applicants": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "foundation": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "goodbye": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "support": 6 |
| }, |
| "grading_gpa": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "greeting": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "inclusive_education": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "international_exams": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "language_cert_discounts": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "library": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "location": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "master_admission": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "master_programs": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "medical_center": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "military": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "missed_exam": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "other_discounts": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "passing_score": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "payment_details": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "phd_admission": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "portals": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "practice": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "programs_list": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "retake_fx": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "schedule": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "state_grant": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "stipend": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "student_clubs": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "study_forms": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "thanks": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "transfer_restore": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "support": 6 |
| }, |
| "tuition_bachelor": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "tuition_postgrad": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "turan_umiti_grant": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "vacant_grant": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| } |
| }, |
| "confusions": [ |
| { |
| "true": "greeting", |
| "pred": "goodbye", |
| "count": 1 |
| }, |
| { |
| "true": "capabilities", |
| "pred": "international_exams", |
| "count": 1 |
| }, |
| { |
| "true": "admission_steps", |
| "pred": "transfer_restore", |
| "count": 1 |
| }, |
| { |
| "true": "ent_subjects", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "passing_score", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "creative_exam", |
| "pred": "appeal", |
| "count": 1 |
| }, |
| { |
| "true": "college_graduates", |
| "pred": "missed_exam", |
| "count": 1 |
| }, |
| { |
| "true": "programs_list", |
| "pred": "medical_center", |
| "count": 1 |
| }, |
| { |
| "true": "programs_list", |
| "pred": "transfer_restore", |
| "count": 1 |
| }, |
| { |
| "true": "study_forms", |
| "pred": "goodbye", |
| "count": 1 |
| } |
| ] |
| }, |
| "ood_test": { |
| "overall": { |
| "value": 0.7692307692307693, |
| "ci95": [ |
| 0.6644424208413748, |
| 0.8487449120105162 |
| ], |
| "n": 78, |
| "successes": 60 |
| }, |
| "ood_rejected": { |
| "value": 0.7692307692307693, |
| "ci95": [ |
| 0.6644424208413748, |
| 0.8487449120105162 |
| ], |
| "n": 78, |
| "successes": 60 |
| }, |
| "by_group": { |
| "near_domain": { |
| "value": 0.5652173913043478, |
| "ci95": [ |
| 0.3681137494517215, |
| 0.7436536478893558 |
| ], |
| "n": 23, |
| "successes": 13 |
| }, |
| "nonsense": { |
| "value": 0.8666666666666667, |
| "ci95": [ |
| 0.6211801694601944, |
| 0.9626387412357151 |
| ], |
| "n": 15, |
| "successes": 13 |
| }, |
| "offtopic": { |
| "value": 0.85, |
| "ci95": [ |
| 0.7092767550401498, |
| 0.9293881232389256 |
| ], |
| "n": 40, |
| "successes": 34 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.7777777777777778, |
| "ci95": [ |
| 0.5478541549399641, |
| 0.9099907195415445 |
| ], |
| "n": 18, |
| "successes": 14 |
| }, |
| "kk": { |
| "value": 0.8, |
| "ci95": [ |
| 0.4901624691221323, |
| 0.9433178490622331 |
| ], |
| "n": 10, |
| "successes": 8 |
| }, |
| "ru": { |
| "value": 0.76, |
| "ci95": [ |
| 0.6258731612803865, |
| 0.8570260866267341 |
| ], |
| "n": 50, |
| "successes": 38 |
| } |
| } |
| }, |
| "external": { |
| "overall": { |
| "value": 0.6267942583732058, |
| "ci95": [ |
| 0.5594889825720564, |
| 0.6895226539698576 |
| ], |
| "n": 209, |
| "successes": 131 |
| }, |
| "in_domain_correct": { |
| "value": 0.5142857142857142, |
| "ci95": [ |
| 0.43222650430712206, |
| 0.5955818899563214 |
| ], |
| "n": 140, |
| "successes": 72 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.7642857142857142, |
| "ci95": [ |
| 0.6875066304583293, |
| 0.8269486634153786 |
| ], |
| "n": 140, |
| "successes": 107 |
| }, |
| "ood_rejected": { |
| "value": 0.855072463768116, |
| "ci95": [ |
| 0.7533647830164287, |
| 0.9193290475650187 |
| ], |
| "n": 69, |
| "successes": 59 |
| }, |
| "by_group": { |
| "Caspian": { |
| "value": 0.6129032258064516, |
| "ci95": [ |
| 0.43824108487672686, |
| 0.7626689867666228 |
| ], |
| "n": 31, |
| "successes": 19 |
| }, |
| "EKTU": { |
| "value": 0.5714285714285714, |
| "ci95": [ |
| 0.25045836275303557, |
| 0.8417801460075546 |
| ], |
| "n": 7, |
| "successes": 4 |
| }, |
| "KIMEP": { |
| "value": 0.7857142857142857, |
| "ci95": [ |
| 0.5241076920169465, |
| 0.9242861334670647 |
| ], |
| "n": 14, |
| "successes": 11 |
| }, |
| "KarTU": { |
| "value": 0.3076923076923077, |
| "ci95": [ |
| 0.12680703567070326, |
| 0.5763065701492199 |
| ], |
| "n": 13, |
| "successes": 4 |
| }, |
| "NU": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "SDU": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "Satbayev": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "TorU": { |
| "value": 0.5851063829787234, |
| "ci95": [ |
| 0.4840802044979575, |
| 0.6794496547553848 |
| ], |
| "n": 94, |
| "successes": 55 |
| }, |
| "Yasawi": { |
| "value": 0.71875, |
| "ci95": [ |
| 0.5462549043349592, |
| 0.8443541957679932 |
| ], |
| "n": 32, |
| "successes": 23 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.5319148936170213, |
| "ci95": [ |
| 0.39232383116617336, |
| 0.6666831301998322 |
| ], |
| "n": 47, |
| "successes": 25 |
| }, |
| "kk": { |
| "value": 0.6071428571428571, |
| "ci95": [ |
| 0.42409044159980325, |
| 0.7643431355917387 |
| ], |
| "n": 28, |
| "successes": 17 |
| }, |
| "ru": { |
| "value": 0.664179104477612, |
| "ci95": [ |
| 0.58062954268048, |
| 0.738577757917602 |
| ], |
| "n": 134, |
| "successes": 89 |
| } |
| } |
| }, |
| "scenarios": { |
| "overall": { |
| "value": 0.8461538461538461, |
| "ci95": [ |
| 0.7394468026298345, |
| 0.9142290597250574 |
| ], |
| "n": 65, |
| "successes": 55 |
| }, |
| "in_domain_correct": { |
| "value": 0.8360655737704918, |
| "ci95": [ |
| 0.7238716136704956, |
| 0.9084398843643947 |
| ], |
| "n": 61, |
| "successes": 51 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.9344262295081968, |
| "ci95": [ |
| 0.8431719822578765, |
| 0.9742062956896644 |
| ], |
| "n": 61, |
| "successes": 57 |
| }, |
| "ood_rejected": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "by_group": { |
| "college_graduate": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "exchange_student": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "first_year_student": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "foreign_applicant": { |
| "value": 0.6666666666666666, |
| "ci95": [ |
| 0.29999331299474324, |
| 0.9032285897864654 |
| ], |
| "n": 6, |
| "successes": 4 |
| }, |
| "master_applicant": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "parent": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "phd_applicant": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "random_visitor": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "school_graduate": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "student_failed_exam": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5655175313406071, |
| 1.0 |
| ], |
| "n": 5, |
| "successes": 5 |
| }, |
| "student_paid_to_grant": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| }, |
| "student_special_needs": { |
| "value": 0.5, |
| "ci95": [ |
| 0.15003898774401986, |
| 0.8499610122559802 |
| ], |
| "n": 4, |
| "successes": 2 |
| }, |
| "student_with_family": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.8, |
| "ci95": [ |
| 0.4901624691221323, |
| 0.9433178490622331 |
| ], |
| "n": 10, |
| "successes": 8 |
| }, |
| "kk": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "ru": { |
| "value": 0.8367346938775511, |
| "ci95": [ |
| 0.7096366145094091, |
| 0.9148730167279078 |
| ], |
| "n": 49, |
| "successes": 41 |
| } |
| } |
| }, |
| "C": 1000 |
| }, |
| "e5_finetuned_meanpool": { |
| "cv_accuracy": 0.7487135506003431, |
| "threshold_selection": { |
| "score": 0.6380789022298456, |
| "accepted_correct": 0.548885077186964, |
| "rejected_ood": 0.7272727272727273 |
| }, |
| "threshold": 0.3792423903942108, |
| "test": { |
| "overall": { |
| "value": 0.7012578616352201, |
| "ci95": [ |
| 0.6487929171843486, |
| 0.748918429405122 |
| ], |
| "n": 318, |
| "successes": 223 |
| }, |
| "in_domain_correct": { |
| "value": 0.7012578616352201, |
| "ci95": [ |
| 0.6487929171843486, |
| 0.748918429405122 |
| ], |
| "n": 318, |
| "successes": 223 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.8113207547169812, |
| "ci95": [ |
| 0.7646986465268895, |
| 0.8505110927286981 |
| ], |
| "n": 318, |
| "successes": 258 |
| }, |
| "by_group": { |
| "long": { |
| "value": 0.7358490566037735, |
| "ci95": [ |
| 0.6041933547921376, |
| 0.8356264614481239 |
| ], |
| "n": 53, |
| "successes": 39 |
| }, |
| "mixed": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6456695613241317, |
| 1.0 |
| ], |
| "n": 7, |
| "successes": 7 |
| }, |
| "plain": { |
| "value": 0.8383838383838383, |
| "ci95": [ |
| 0.7534877129503085, |
| 0.898000516882509 |
| ], |
| "n": 99, |
| "successes": 83 |
| }, |
| "slang": { |
| "value": 0.5849056603773585, |
| "ci95": [ |
| 0.450944220784094, |
| 0.7073909130624002 |
| ], |
| "n": 53, |
| "successes": 31 |
| }, |
| "translit": { |
| "value": 0.3584905660377358, |
| "ci95": [ |
| 0.24301952474377947, |
| 0.4930885855120634 |
| ], |
| "n": 53, |
| "successes": 19 |
| }, |
| "typo": { |
| "value": 0.8301886792452831, |
| "ci95": [ |
| 0.7077471960878099, |
| 0.9080005466485566 |
| ], |
| "n": 53, |
| "successes": 44 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.8490566037735849, |
| "ci95": [ |
| 0.7294575061422911, |
| 0.9214758218932962 |
| ], |
| "n": 53, |
| "successes": 45 |
| }, |
| "kk": { |
| "value": 0.8490566037735849, |
| "ci95": [ |
| 0.7294575061422911, |
| 0.9214758218932962 |
| ], |
| "n": 53, |
| "successes": 45 |
| }, |
| "ru": { |
| "value": 0.6273584905660378, |
| "ci95": [ |
| 0.560548370726413, |
| 0.6896352607217873 |
| ], |
| "n": 212, |
| "successes": 133 |
| } |
| }, |
| "macro_f1_no_threshold": 0.8062052727147064, |
| "per_intent": { |
| "about_university": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "academic_calendar": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "academic_integrity": { |
| "precision": 0.4, |
| "recall": 0.3333333333333333, |
| "f1": 0.3636363636363636, |
| "support": 6 |
| }, |
| "academic_leave": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "academic_mobility": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "admission_deadlines": { |
| "precision": 0.8, |
| "recall": 0.6666666666666666, |
| "f1": 0.7272727272727272, |
| "support": 6 |
| }, |
| "admission_documents": { |
| "precision": 0.5555555555555556, |
| "recall": 0.8333333333333334, |
| "f1": 0.6666666666666667, |
| "support": 6 |
| }, |
| "admission_steps": { |
| "precision": 0.6, |
| "recall": 0.5, |
| "f1": 0.5454545454545454, |
| "support": 6 |
| }, |
| "appeal": { |
| "precision": 0.625, |
| "recall": 0.8333333333333334, |
| "f1": 0.7142857142857143, |
| "support": 6 |
| }, |
| "capabilities": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "career_center": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "certificates": { |
| "precision": 0.75, |
| "recall": 0.5, |
| "f1": 0.6, |
| "support": 6 |
| }, |
| "college_graduates": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "contacts": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "creative_exam": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "dormitory": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "ent_discounts": { |
| "precision": 0.75, |
| "recall": 0.5, |
| "f1": 0.6, |
| "support": 6 |
| }, |
| "ent_subjects": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "support": 6 |
| }, |
| "faculties": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "foreign_applicants": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "foundation": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "goodbye": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "grading_gpa": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "greeting": { |
| "precision": 0.5, |
| "recall": 0.8333333333333334, |
| "f1": 0.625, |
| "support": 6 |
| }, |
| "inclusive_education": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "international_exams": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "language_cert_discounts": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "library": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "location": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "master_admission": { |
| "precision": 1.0, |
| "recall": 0.5, |
| "f1": 0.6666666666666666, |
| "support": 6 |
| }, |
| "master_programs": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "support": 6 |
| }, |
| "medical_center": { |
| "precision": 0.6666666666666666, |
| "recall": 0.6666666666666666, |
| "f1": 0.6666666666666666, |
| "support": 6 |
| }, |
| "military": { |
| "precision": 0.8, |
| "recall": 0.6666666666666666, |
| "f1": 0.7272727272727272, |
| "support": 6 |
| }, |
| "missed_exam": { |
| "precision": 0.625, |
| "recall": 0.8333333333333334, |
| "f1": 0.7142857142857143, |
| "support": 6 |
| }, |
| "other_discounts": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "passing_score": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "payment_details": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "phd_admission": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "portals": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "practice": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "programs_list": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "retake_fx": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "schedule": { |
| "precision": 1.0, |
| "recall": 0.5, |
| "f1": 0.6666666666666666, |
| "support": 6 |
| }, |
| "state_grant": { |
| "precision": 0.5, |
| "recall": 0.16666666666666666, |
| "f1": 0.25, |
| "support": 6 |
| }, |
| "stipend": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "student_clubs": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "study_forms": { |
| "precision": 0.6666666666666666, |
| "recall": 0.6666666666666666, |
| "f1": 0.6666666666666666, |
| "support": 6 |
| }, |
| "thanks": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "transfer_restore": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "tuition_bachelor": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "tuition_postgrad": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "turan_umiti_grant": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "vacant_grant": { |
| "precision": 0.625, |
| "recall": 0.8333333333333334, |
| "f1": 0.7142857142857143, |
| "support": 6 |
| } |
| }, |
| "confusions": [ |
| { |
| "true": "state_grant", |
| "pred": "vacant_grant", |
| "count": 3 |
| }, |
| { |
| "true": "thanks", |
| "pred": "greeting", |
| "count": 2 |
| }, |
| { |
| "true": "admission_deadlines", |
| "pred": "admission_documents", |
| "count": 2 |
| }, |
| { |
| "true": "greeting", |
| "pred": "admission_steps", |
| "count": 1 |
| }, |
| { |
| "true": "goodbye", |
| "pred": "greeting", |
| "count": 1 |
| }, |
| { |
| "true": "capabilities", |
| "pred": "greeting", |
| "count": 1 |
| }, |
| { |
| "true": "capabilities", |
| "pred": "ent_subjects", |
| "count": 1 |
| }, |
| { |
| "true": "admission_steps", |
| "pred": "study_forms", |
| "count": 1 |
| }, |
| { |
| "true": "admission_steps", |
| "pred": "foundation", |
| "count": 1 |
| }, |
| { |
| "true": "admission_steps", |
| "pred": "admission_deadlines", |
| "count": 1 |
| } |
| ] |
| }, |
| "ood_test": { |
| "overall": { |
| "value": 0.6282051282051282, |
| "ci95": [ |
| 0.5173105369311474, |
| 0.7270643834745151 |
| ], |
| "n": 78, |
| "successes": 49 |
| }, |
| "ood_rejected": { |
| "value": 0.6282051282051282, |
| "ci95": [ |
| 0.5173105369311474, |
| 0.7270643834745151 |
| ], |
| "n": 78, |
| "successes": 49 |
| }, |
| "by_group": { |
| "near_domain": { |
| "value": 0.5652173913043478, |
| "ci95": [ |
| 0.3681137494517215, |
| 0.7436536478893558 |
| ], |
| "n": 23, |
| "successes": 13 |
| }, |
| "nonsense": { |
| "value": 0.6, |
| "ci95": [ |
| 0.35746829954344506, |
| 0.8017550397372577 |
| ], |
| "n": 15, |
| "successes": 9 |
| }, |
| "offtopic": { |
| "value": 0.675, |
| "ci95": [ |
| 0.5201774606620777, |
| 0.7991549784774601 |
| ], |
| "n": 40, |
| "successes": 27 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.4444444444444444, |
| "ci95": [ |
| 0.24559518003796732, |
| 0.6628358450657309 |
| ], |
| "n": 18, |
| "successes": 8 |
| }, |
| "kk": { |
| "value": 0.6, |
| "ci95": [ |
| 0.3126737679211953, |
| 0.831819671473593 |
| ], |
| "n": 10, |
| "successes": 6 |
| }, |
| "ru": { |
| "value": 0.7, |
| "ci95": [ |
| 0.5624964942316114, |
| 0.8089644656969429 |
| ], |
| "n": 50, |
| "successes": 35 |
| } |
| } |
| }, |
| "external": { |
| "overall": { |
| "value": 0.6267942583732058, |
| "ci95": [ |
| 0.5594889825720564, |
| 0.6895226539698576 |
| ], |
| "n": 209, |
| "successes": 131 |
| }, |
| "in_domain_correct": { |
| "value": 0.5428571428571428, |
| "ci95": [ |
| 0.46029509949444614, |
| 0.6231300832958848 |
| ], |
| "n": 140, |
| "successes": 76 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.7357142857142858, |
| "ci95": [ |
| 0.6570842789810405, |
| 0.8017542263657802 |
| ], |
| "n": 140, |
| "successes": 103 |
| }, |
| "ood_rejected": { |
| "value": 0.7971014492753623, |
| "ci95": [ |
| 0.687759596598563, |
| 0.8751066698063215 |
| ], |
| "n": 69, |
| "successes": 55 |
| }, |
| "by_group": { |
| "Caspian": { |
| "value": 0.45161290322580644, |
| "ci95": [ |
| 0.29161735871855426, |
| 0.6222783248628674 |
| ], |
| "n": 31, |
| "successes": 14 |
| }, |
| "EKTU": { |
| "value": 0.5714285714285714, |
| "ci95": [ |
| 0.25045836275303557, |
| 0.8417801460075546 |
| ], |
| "n": 7, |
| "successes": 4 |
| }, |
| "KIMEP": { |
| "value": 0.7142857142857143, |
| "ci95": [ |
| 0.45350915474002584, |
| 0.8827862143729825 |
| ], |
| "n": 14, |
| "successes": 10 |
| }, |
| "KarTU": { |
| "value": 0.38461538461538464, |
| "ci95": [ |
| 0.17709707685632237, |
| 0.6447710866356315 |
| ], |
| "n": 13, |
| "successes": 5 |
| }, |
| "NU": { |
| "value": 0.6666666666666666, |
| "ci95": [ |
| 0.29999331299474324, |
| 0.9032285897864654 |
| ], |
| "n": 6, |
| "successes": 4 |
| }, |
| "SDU": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "Satbayev": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "TorU": { |
| "value": 0.6276595744680851, |
| "ci95": [ |
| 0.5267266542388092, |
| 0.7185681346412041 |
| ], |
| "n": 94, |
| "successes": 59 |
| }, |
| "Yasawi": { |
| "value": 0.71875, |
| "ci95": [ |
| 0.5462549043349592, |
| 0.8443541957679932 |
| ], |
| "n": 32, |
| "successes": 23 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.7446808510638298, |
| "ci95": [ |
| 0.6049182435213335, |
| 0.8474684602847101 |
| ], |
| "n": 47, |
| "successes": 35 |
| }, |
| "kk": { |
| "value": 0.6428571428571429, |
| "ci95": [ |
| 0.45830306373230584, |
| 0.792941705856417 |
| ], |
| "n": 28, |
| "successes": 18 |
| }, |
| "ru": { |
| "value": 0.582089552238806, |
| "ci95": [ |
| 0.49743311710055926, |
| 0.6621705331984817 |
| ], |
| "n": 134, |
| "successes": 78 |
| } |
| } |
| }, |
| "scenarios": { |
| "overall": { |
| "value": 0.8615384615384616, |
| "ci95": [ |
| 0.7573190265740312, |
| 0.9254090963299672 |
| ], |
| "n": 65, |
| "successes": 56 |
| }, |
| "in_domain_correct": { |
| "value": 0.8524590163934426, |
| "ci95": [ |
| 0.7427676460474365, |
| 0.9203883153062286 |
| ], |
| "n": 61, |
| "successes": 52 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.9180327868852459, |
| "ci95": [ |
| 0.8220533810108364, |
| 0.9644804336179293 |
| ], |
| "n": 61, |
| "successes": 56 |
| }, |
| "ood_rejected": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "by_group": { |
| "college_graduate": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "exchange_student": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| }, |
| "first_year_student": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "foreign_applicant": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "master_applicant": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5655175313406071, |
| 1.0 |
| ], |
| "n": 5, |
| "successes": 5 |
| }, |
| "parent": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "phd_applicant": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "random_visitor": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "school_graduate": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "student_failed_exam": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "student_paid_to_grant": { |
| "value": 0.5, |
| "ci95": [ |
| 0.15003898774401986, |
| 0.8499610122559802 |
| ], |
| "n": 4, |
| "successes": 2 |
| }, |
| "student_special_needs": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| }, |
| "student_with_family": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.9, |
| "ci95": [ |
| 0.5958499704752279, |
| 0.982123787103926 |
| ], |
| "n": 10, |
| "successes": 9 |
| }, |
| "kk": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "ru": { |
| "value": 0.8367346938775511, |
| "ci95": [ |
| 0.7096366145094091, |
| 0.9148730167279078 |
| ], |
| "n": 49, |
| "successes": 41 |
| } |
| } |
| }, |
| "train_minutes_cv_and_final": 3.9026201049486797 |
| } |
| }, |
| "quantization": { |
| "encoder/fp32": { |
| "size_mb": 448.47027492523193, |
| "session_rss_mb": 515.828125, |
| "latency_ms": { |
| "p50": 12.38074050013438, |
| "p95": 22.699632999990623, |
| "p99": 30.91484616996013, |
| "mean": 14.162123775677468, |
| "n": 954 |
| }, |
| "top1_agreement_vs_fp32": 1.0, |
| "cos_mean": 1.0, |
| "cos_min": 0.999999463558197, |
| "eval": { |
| "cv_accuracy": 0.8027444253859348, |
| "threshold": 0.731022679863181, |
| "test": { |
| "overall": { |
| "value": 0.6949685534591195, |
| "ci95": [ |
| 0.6422860107063452, |
| 0.7429968563022046 |
| ], |
| "n": 318, |
| "successes": 221 |
| }, |
| "in_domain_correct": { |
| "value": 0.6949685534591195, |
| "ci95": [ |
| 0.6422860107063452, |
| 0.7429968563022046 |
| ], |
| "n": 318, |
| "successes": 221 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.8805031446540881, |
| "ci95": [ |
| 0.8402335419115275, |
| 0.9116894727341905 |
| ], |
| "n": 318, |
| "successes": 280 |
| }, |
| "by_group": { |
| "long": { |
| "value": 0.6981132075471698, |
| "ci95": [ |
| 0.5646316722203186, |
| 0.8048169734215012 |
| ], |
| "n": 53, |
| "successes": 37 |
| }, |
| "mixed": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6456695613241317, |
| 1.0 |
| ], |
| "n": 7, |
| "successes": 7 |
| }, |
| "plain": { |
| "value": 0.8080808080808081, |
| "ci95": [ |
| 0.7195963930846633, |
| 0.8735496072108871 |
| ], |
| "n": 99, |
| "successes": 80 |
| }, |
| "slang": { |
| "value": 0.6415094339622641, |
| "ci95": [ |
| 0.5069114144879365, |
| 0.7569804752562204 |
| ], |
| "n": 53, |
| "successes": 34 |
| }, |
| "translit": { |
| "value": 0.32075471698113206, |
| "ci95": [ |
| 0.2109221400637985, |
| 0.4548147995936025 |
| ], |
| "n": 53, |
| "successes": 17 |
| }, |
| "typo": { |
| "value": 0.8679245283018868, |
| "ci95": [ |
| 0.751597445822794, |
| 0.9345214675120143 |
| ], |
| "n": 53, |
| "successes": 46 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.8679245283018868, |
| "ci95": [ |
| 0.751597445822794, |
| 0.9345214675120143 |
| ], |
| "n": 53, |
| "successes": 46 |
| }, |
| "kk": { |
| "value": 0.7735849056603774, |
| "ci95": [ |
| 0.644737296378973, |
| 0.8654536904597305 |
| ], |
| "n": 53, |
| "successes": 41 |
| }, |
| "ru": { |
| "value": 0.6320754716981132, |
| "ci95": [ |
| 0.5653473327892489, |
| 0.6941023590829587 |
| ], |
| "n": 212, |
| "successes": 134 |
| } |
| }, |
| "macro_f1_no_threshold": 0.8839966637136447, |
| "per_intent": { |
| "about_university": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "academic_calendar": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "academic_integrity": { |
| "precision": 0.5, |
| "recall": 0.8333333333333334, |
| "f1": 0.625, |
| "support": 6 |
| }, |
| "academic_leave": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "academic_mobility": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "admission_deadlines": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "admission_documents": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "support": 6 |
| }, |
| "admission_steps": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "appeal": { |
| "precision": 0.625, |
| "recall": 0.8333333333333334, |
| "f1": 0.7142857142857143, |
| "support": 6 |
| }, |
| "capabilities": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "career_center": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "certificates": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "college_graduates": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "contacts": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "creative_exam": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "dormitory": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "ent_discounts": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "ent_subjects": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "faculties": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "foreign_applicants": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "foundation": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "goodbye": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "support": 6 |
| }, |
| "grading_gpa": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "greeting": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "inclusive_education": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "international_exams": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "language_cert_discounts": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "library": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "location": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "master_admission": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "master_programs": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "medical_center": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "military": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "missed_exam": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "other_discounts": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "passing_score": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "payment_details": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "phd_admission": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "portals": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "practice": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "programs_list": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "retake_fx": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "schedule": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "state_grant": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "stipend": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "student_clubs": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "study_forms": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "thanks": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "transfer_restore": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "support": 6 |
| }, |
| "tuition_bachelor": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "tuition_postgrad": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "turan_umiti_grant": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "vacant_grant": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| } |
| }, |
| "confusions": [ |
| { |
| "true": "greeting", |
| "pred": "goodbye", |
| "count": 1 |
| }, |
| { |
| "true": "capabilities", |
| "pred": "international_exams", |
| "count": 1 |
| }, |
| { |
| "true": "admission_steps", |
| "pred": "transfer_restore", |
| "count": 1 |
| }, |
| { |
| "true": "ent_subjects", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "passing_score", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "creative_exam", |
| "pred": "appeal", |
| "count": 1 |
| }, |
| { |
| "true": "college_graduates", |
| "pred": "missed_exam", |
| "count": 1 |
| }, |
| { |
| "true": "programs_list", |
| "pred": "medical_center", |
| "count": 1 |
| }, |
| { |
| "true": "programs_list", |
| "pred": "transfer_restore", |
| "count": 1 |
| }, |
| { |
| "true": "study_forms", |
| "pred": "goodbye", |
| "count": 1 |
| } |
| ] |
| }, |
| "ood_test": { |
| "overall": { |
| "value": 0.7692307692307693, |
| "ci95": [ |
| 0.6644424208413748, |
| 0.8487449120105162 |
| ], |
| "n": 78, |
| "successes": 60 |
| }, |
| "ood_rejected": { |
| "value": 0.7692307692307693, |
| "ci95": [ |
| 0.6644424208413748, |
| 0.8487449120105162 |
| ], |
| "n": 78, |
| "successes": 60 |
| }, |
| "by_group": { |
| "near_domain": { |
| "value": 0.5652173913043478, |
| "ci95": [ |
| 0.3681137494517215, |
| 0.7436536478893558 |
| ], |
| "n": 23, |
| "successes": 13 |
| }, |
| "nonsense": { |
| "value": 0.8666666666666667, |
| "ci95": [ |
| 0.6211801694601944, |
| 0.9626387412357151 |
| ], |
| "n": 15, |
| "successes": 13 |
| }, |
| "offtopic": { |
| "value": 0.85, |
| "ci95": [ |
| 0.7092767550401498, |
| 0.9293881232389256 |
| ], |
| "n": 40, |
| "successes": 34 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.7777777777777778, |
| "ci95": [ |
| 0.5478541549399641, |
| 0.9099907195415445 |
| ], |
| "n": 18, |
| "successes": 14 |
| }, |
| "kk": { |
| "value": 0.8, |
| "ci95": [ |
| 0.4901624691221323, |
| 0.9433178490622331 |
| ], |
| "n": 10, |
| "successes": 8 |
| }, |
| "ru": { |
| "value": 0.76, |
| "ci95": [ |
| 0.6258731612803865, |
| 0.8570260866267341 |
| ], |
| "n": 50, |
| "successes": 38 |
| } |
| } |
| }, |
| "external": { |
| "overall": { |
| "value": 0.6267942583732058, |
| "ci95": [ |
| 0.5594889825720564, |
| 0.6895226539698576 |
| ], |
| "n": 209, |
| "successes": 131 |
| }, |
| "in_domain_correct": { |
| "value": 0.5142857142857142, |
| "ci95": [ |
| 0.43222650430712206, |
| 0.5955818899563214 |
| ], |
| "n": 140, |
| "successes": 72 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.7642857142857142, |
| "ci95": [ |
| 0.6875066304583293, |
| 0.8269486634153786 |
| ], |
| "n": 140, |
| "successes": 107 |
| }, |
| "ood_rejected": { |
| "value": 0.855072463768116, |
| "ci95": [ |
| 0.7533647830164287, |
| 0.9193290475650187 |
| ], |
| "n": 69, |
| "successes": 59 |
| }, |
| "by_group": { |
| "Caspian": { |
| "value": 0.6129032258064516, |
| "ci95": [ |
| 0.43824108487672686, |
| 0.7626689867666228 |
| ], |
| "n": 31, |
| "successes": 19 |
| }, |
| "EKTU": { |
| "value": 0.5714285714285714, |
| "ci95": [ |
| 0.25045836275303557, |
| 0.8417801460075546 |
| ], |
| "n": 7, |
| "successes": 4 |
| }, |
| "KIMEP": { |
| "value": 0.7857142857142857, |
| "ci95": [ |
| 0.5241076920169465, |
| 0.9242861334670647 |
| ], |
| "n": 14, |
| "successes": 11 |
| }, |
| "KarTU": { |
| "value": 0.3076923076923077, |
| "ci95": [ |
| 0.12680703567070326, |
| 0.5763065701492199 |
| ], |
| "n": 13, |
| "successes": 4 |
| }, |
| "NU": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "SDU": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "Satbayev": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "TorU": { |
| "value": 0.5851063829787234, |
| "ci95": [ |
| 0.4840802044979575, |
| 0.6794496547553848 |
| ], |
| "n": 94, |
| "successes": 55 |
| }, |
| "Yasawi": { |
| "value": 0.71875, |
| "ci95": [ |
| 0.5462549043349592, |
| 0.8443541957679932 |
| ], |
| "n": 32, |
| "successes": 23 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.5319148936170213, |
| "ci95": [ |
| 0.39232383116617336, |
| 0.6666831301998322 |
| ], |
| "n": 47, |
| "successes": 25 |
| }, |
| "kk": { |
| "value": 0.6071428571428571, |
| "ci95": [ |
| 0.42409044159980325, |
| 0.7643431355917387 |
| ], |
| "n": 28, |
| "successes": 17 |
| }, |
| "ru": { |
| "value": 0.664179104477612, |
| "ci95": [ |
| 0.58062954268048, |
| 0.738577757917602 |
| ], |
| "n": 134, |
| "successes": 89 |
| } |
| } |
| }, |
| "scenarios": { |
| "overall": { |
| "value": 0.8461538461538461, |
| "ci95": [ |
| 0.7394468026298345, |
| 0.9142290597250574 |
| ], |
| "n": 65, |
| "successes": 55 |
| }, |
| "in_domain_correct": { |
| "value": 0.8360655737704918, |
| "ci95": [ |
| 0.7238716136704956, |
| 0.9084398843643947 |
| ], |
| "n": 61, |
| "successes": 51 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.9344262295081968, |
| "ci95": [ |
| 0.8431719822578765, |
| 0.9742062956896644 |
| ], |
| "n": 61, |
| "successes": 57 |
| }, |
| "ood_rejected": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "by_group": { |
| "college_graduate": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "exchange_student": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "first_year_student": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "foreign_applicant": { |
| "value": 0.6666666666666666, |
| "ci95": [ |
| 0.29999331299474324, |
| 0.9032285897864654 |
| ], |
| "n": 6, |
| "successes": 4 |
| }, |
| "master_applicant": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "parent": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "phd_applicant": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "random_visitor": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "school_graduate": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "student_failed_exam": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5655175313406071, |
| 1.0 |
| ], |
| "n": 5, |
| "successes": 5 |
| }, |
| "student_paid_to_grant": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| }, |
| "student_special_needs": { |
| "value": 0.5, |
| "ci95": [ |
| 0.15003898774401986, |
| 0.8499610122559802 |
| ], |
| "n": 4, |
| "successes": 2 |
| }, |
| "student_with_family": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.8, |
| "ci95": [ |
| 0.4901624691221323, |
| 0.9433178490622331 |
| ], |
| "n": 10, |
| "successes": 8 |
| }, |
| "kk": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "ru": { |
| "value": 0.8367346938775511, |
| "ci95": [ |
| 0.7096366145094091, |
| 0.9148730167279078 |
| ], |
| "n": 49, |
| "successes": 41 |
| } |
| } |
| } |
| } |
| }, |
| "encoder/int8_full": { |
| "size_mb": 112.63661861419678, |
| "session_rss_mb": 150.734375, |
| "latency_ms": { |
| "p50": 7.2485230000438605, |
| "p95": 14.647330950049303, |
| "p99": 18.93137950997016, |
| "mean": 8.278972968551122, |
| "n": 954 |
| }, |
| "top1_agreement_vs_fp32": 0.9534762153685311, |
| "cos_mean": 0.9882081151008606, |
| "cos_min": 0.9554386734962463, |
| "eval": { |
| "cv_accuracy": 0.8027444253859348, |
| "threshold": 0.731022679863181, |
| "test": { |
| "overall": { |
| "value": 0.6257861635220126, |
| "ci95": [ |
| 0.5713946157227733, |
| 0.6771749758956459 |
| ], |
| "n": 318, |
| "successes": 199 |
| }, |
| "in_domain_correct": { |
| "value": 0.6257861635220126, |
| "ci95": [ |
| 0.5713946157227733, |
| 0.6771749758956459 |
| ], |
| "n": 318, |
| "successes": 199 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.8679245283018868, |
| "ci95": [ |
| 0.8262836944014963, |
| 0.9007823610823799 |
| ], |
| "n": 318, |
| "successes": 276 |
| }, |
| "by_group": { |
| "long": { |
| "value": 0.6037735849056604, |
| "ci95": [ |
| 0.4694147856894524, |
| 0.7241059334562627 |
| ], |
| "n": 53, |
| "successes": 32 |
| }, |
| "mixed": { |
| "value": 0.8571428571428571, |
| "ci95": [ |
| 0.486872167860775, |
| 0.9743203759421759 |
| ], |
| "n": 7, |
| "successes": 6 |
| }, |
| "plain": { |
| "value": 0.7575757575757576, |
| "ci95": [ |
| 0.6645718441525184, |
| 0.831337106914253 |
| ], |
| "n": 99, |
| "successes": 75 |
| }, |
| "slang": { |
| "value": 0.5471698113207547, |
| "ci95": [ |
| 0.414539753871568, |
| 0.6734242093764842 |
| ], |
| "n": 53, |
| "successes": 29 |
| }, |
| "translit": { |
| "value": 0.2830188679245283, |
| "ci95": [ |
| 0.17966290379767413, |
| 0.41570286526128497 |
| ], |
| "n": 53, |
| "successes": 15 |
| }, |
| "typo": { |
| "value": 0.7924528301886793, |
| "ci95": [ |
| 0.6654225518770907, |
| 0.879954020260834 |
| ], |
| "n": 53, |
| "successes": 42 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.7924528301886793, |
| "ci95": [ |
| 0.6654225518770907, |
| 0.879954020260834 |
| ], |
| "n": 53, |
| "successes": 42 |
| }, |
| "kk": { |
| "value": 0.7358490566037735, |
| "ci95": [ |
| 0.6041933547921376, |
| 0.8356264614481239 |
| ], |
| "n": 53, |
| "successes": 39 |
| }, |
| "ru": { |
| "value": 0.5566037735849056, |
| "ci95": [ |
| 0.48931370738943414, |
| 0.6218790176986547 |
| ], |
| "n": 212, |
| "successes": 118 |
| } |
| }, |
| "macro_f1_no_threshold": 0.8727878805774711, |
| "per_intent": { |
| "about_university": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "academic_calendar": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "academic_integrity": { |
| "precision": 0.5454545454545454, |
| "recall": 1.0, |
| "f1": 0.7058823529411764, |
| "support": 6 |
| }, |
| "academic_leave": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "academic_mobility": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "admission_deadlines": { |
| "precision": 0.625, |
| "recall": 0.8333333333333334, |
| "f1": 0.7142857142857143, |
| "support": 6 |
| }, |
| "admission_documents": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "admission_steps": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "appeal": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "support": 6 |
| }, |
| "capabilities": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "career_center": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "certificates": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "college_graduates": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "contacts": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "creative_exam": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "dormitory": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "ent_discounts": { |
| "precision": 0.625, |
| "recall": 0.8333333333333334, |
| "f1": 0.7142857142857143, |
| "support": 6 |
| }, |
| "ent_subjects": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "faculties": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "foreign_applicants": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "foundation": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "goodbye": { |
| "precision": 0.46153846153846156, |
| "recall": 1.0, |
| "f1": 0.631578947368421, |
| "support": 6 |
| }, |
| "grading_gpa": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "greeting": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "inclusive_education": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "international_exams": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "language_cert_discounts": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "library": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "location": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "master_admission": { |
| "precision": 0.8, |
| "recall": 0.6666666666666666, |
| "f1": 0.7272727272727272, |
| "support": 6 |
| }, |
| "master_programs": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "medical_center": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "military": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "missed_exam": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "other_discounts": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "passing_score": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "payment_details": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "phd_admission": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "portals": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "practice": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "programs_list": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "retake_fx": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "schedule": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "state_grant": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "stipend": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "student_clubs": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "study_forms": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "thanks": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "transfer_restore": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "tuition_bachelor": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "tuition_postgrad": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "turan_umiti_grant": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "vacant_grant": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| } |
| }, |
| "confusions": [ |
| { |
| "true": "greeting", |
| "pred": "goodbye", |
| "count": 1 |
| }, |
| { |
| "true": "capabilities", |
| "pred": "goodbye", |
| "count": 1 |
| }, |
| { |
| "true": "admission_steps", |
| "pred": "transfer_restore", |
| "count": 1 |
| }, |
| { |
| "true": "ent_subjects", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "passing_score", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "admission_deadlines", |
| "pred": "academic_integrity", |
| "count": 1 |
| }, |
| { |
| "true": "creative_exam", |
| "pred": "appeal", |
| "count": 1 |
| }, |
| { |
| "true": "college_graduates", |
| "pred": "goodbye", |
| "count": 1 |
| }, |
| { |
| "true": "programs_list", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "study_forms", |
| "pred": "goodbye", |
| "count": 1 |
| } |
| ] |
| }, |
| "ood_test": { |
| "overall": { |
| "value": 0.782051282051282, |
| "ci95": [ |
| 0.6783929448085361, |
| 0.859231880083921 |
| ], |
| "n": 78, |
| "successes": 61 |
| }, |
| "ood_rejected": { |
| "value": 0.782051282051282, |
| "ci95": [ |
| 0.6783929448085361, |
| 0.859231880083921 |
| ], |
| "n": 78, |
| "successes": 61 |
| }, |
| "by_group": { |
| "near_domain": { |
| "value": 0.6521739130434783, |
| "ci95": [ |
| 0.4489033522338618, |
| 0.8118872415619854 |
| ], |
| "n": 23, |
| "successes": 15 |
| }, |
| "nonsense": { |
| "value": 0.8, |
| "ci95": [ |
| 0.5481455107594471, |
| 0.9295245070826608 |
| ], |
| "n": 15, |
| "successes": 12 |
| }, |
| "offtopic": { |
| "value": 0.85, |
| "ci95": [ |
| 0.7092767550401498, |
| 0.9293881232389256 |
| ], |
| "n": 40, |
| "successes": 34 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.7777777777777778, |
| "ci95": [ |
| 0.5478541549399641, |
| 0.9099907195415445 |
| ], |
| "n": 18, |
| "successes": 14 |
| }, |
| "kk": { |
| "value": 0.9, |
| "ci95": [ |
| 0.5958499704752279, |
| 0.982123787103926 |
| ], |
| "n": 10, |
| "successes": 9 |
| }, |
| "ru": { |
| "value": 0.76, |
| "ci95": [ |
| 0.6258731612803865, |
| 0.8570260866267341 |
| ], |
| "n": 50, |
| "successes": 38 |
| } |
| } |
| }, |
| "external": { |
| "overall": { |
| "value": 0.6220095693779905, |
| "ci95": [ |
| 0.5546286016538221, |
| 0.6849863693582084 |
| ], |
| "n": 209, |
| "successes": 130 |
| }, |
| "in_domain_correct": { |
| "value": 0.4785714285714286, |
| "ci95": [ |
| 0.3975066132040578, |
| 0.5607807954007769 |
| ], |
| "n": 140, |
| "successes": 67 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.7142857142857143, |
| "ci95": [ |
| 0.6345157084903984, |
| 0.7826102054612566 |
| ], |
| "n": 140, |
| "successes": 100 |
| }, |
| "ood_rejected": { |
| "value": 0.9130434782608695, |
| "ci95": [ |
| 0.8229850157229862, |
| 0.9595363790350239 |
| ], |
| "n": 69, |
| "successes": 63 |
| }, |
| "by_group": { |
| "Caspian": { |
| "value": 0.6129032258064516, |
| "ci95": [ |
| 0.43824108487672686, |
| 0.7626689867666228 |
| ], |
| "n": 31, |
| "successes": 19 |
| }, |
| "EKTU": { |
| "value": 0.5714285714285714, |
| "ci95": [ |
| 0.25045836275303557, |
| 0.8417801460075546 |
| ], |
| "n": 7, |
| "successes": 4 |
| }, |
| "KIMEP": { |
| "value": 0.7142857142857143, |
| "ci95": [ |
| 0.45350915474002584, |
| 0.8827862143729825 |
| ], |
| "n": 14, |
| "successes": 10 |
| }, |
| "KarTU": { |
| "value": 0.23076923076923078, |
| "ci95": [ |
| 0.08179528654269594, |
| 0.5025637616051964 |
| ], |
| "n": 13, |
| "successes": 3 |
| }, |
| "NU": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "SDU": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "Satbayev": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "TorU": { |
| "value": 0.574468085106383, |
| "ci95": [ |
| 0.47353237545666105, |
| 0.6695562513900134 |
| ], |
| "n": 94, |
| "successes": 54 |
| }, |
| "Yasawi": { |
| "value": 0.78125, |
| "ci95": [ |
| 0.6124500620603575, |
| 0.8897616380720099 |
| ], |
| "n": 32, |
| "successes": 25 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.5531914893617021, |
| "ci95": [ |
| 0.4124545973257825, |
| 0.685890338284227 |
| ], |
| "n": 47, |
| "successes": 26 |
| }, |
| "kk": { |
| "value": 0.5714285714285714, |
| "ci95": [ |
| 0.39070785495839777, |
| 0.7349145298359636 |
| ], |
| "n": 28, |
| "successes": 16 |
| }, |
| "ru": { |
| "value": 0.6567164179104478, |
| "ci95": [ |
| 0.5729651365869399, |
| 0.7317327412566836 |
| ], |
| "n": 134, |
| "successes": 88 |
| } |
| } |
| }, |
| "scenarios": { |
| "overall": { |
| "value": 0.7692307692307693, |
| "ci95": [ |
| 0.6535529075284062, |
| 0.8548616520809543 |
| ], |
| "n": 65, |
| "successes": 50 |
| }, |
| "in_domain_correct": { |
| "value": 0.7704918032786885, |
| "ci95": [ |
| 0.6508659477391606, |
| 0.858067697020629 |
| ], |
| "n": 61, |
| "successes": 47 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.9180327868852459, |
| "ci95": [ |
| 0.8220533810108364, |
| 0.9644804336179293 |
| ], |
| "n": 61, |
| "successes": 56 |
| }, |
| "ood_rejected": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| }, |
| "by_group": { |
| "college_graduate": { |
| "value": 0.6, |
| "ci95": [ |
| 0.23072427940911855, |
| 0.8823792268590028 |
| ], |
| "n": 5, |
| "successes": 3 |
| }, |
| "exchange_student": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "first_year_student": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "foreign_applicant": { |
| "value": 0.6666666666666666, |
| "ci95": [ |
| 0.29999331299474324, |
| 0.9032285897864654 |
| ], |
| "n": 6, |
| "successes": 4 |
| }, |
| "master_applicant": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "parent": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "phd_applicant": { |
| "value": 0.5, |
| "ci95": [ |
| 0.15003898774401986, |
| 0.8499610122559802 |
| ], |
| "n": 4, |
| "successes": 2 |
| }, |
| "random_visitor": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "school_graduate": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "student_failed_exam": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "student_paid_to_grant": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| }, |
| "student_special_needs": { |
| "value": 0.5, |
| "ci95": [ |
| 0.15003898774401986, |
| 0.8499610122559802 |
| ], |
| "n": 4, |
| "successes": 2 |
| }, |
| "student_with_family": { |
| "value": 0.5, |
| "ci95": [ |
| 0.15003898774401986, |
| 0.8499610122559802 |
| ], |
| "n": 4, |
| "successes": 2 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.6, |
| "ci95": [ |
| 0.3126737679211953, |
| 0.831819671473593 |
| ], |
| "n": 10, |
| "successes": 6 |
| }, |
| "kk": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "ru": { |
| "value": 0.7755102040816326, |
| "ci95": [ |
| 0.6412121264859051, |
| 0.8697502990718994 |
| ], |
| "n": 49, |
| "successes": 38 |
| } |
| } |
| } |
| } |
| }, |
| "encoder/int8_full_perchannel": { |
| "size_mb": 112.83471584320068, |
| "session_rss_mb": 146.98046875, |
| "latency_ms": { |
| "p50": 8.085354000058942, |
| "p95": 16.675010399921984, |
| "p99": 21.17413892999821, |
| "mean": 9.183721621594007, |
| "n": 954 |
| }, |
| "top1_agreement_vs_fp32": 0.9681129116570831, |
| "cos_mean": 0.9949455261230469, |
| "cos_min": 0.9799907803535461, |
| "eval": { |
| "cv_accuracy": 0.8027444253859348, |
| "threshold": 0.731022679863181, |
| "test": { |
| "overall": { |
| "value": 0.6792452830188679, |
| "ci95": [ |
| 0.626066051198163, |
| 0.7281456168580842 |
| ], |
| "n": 318, |
| "successes": 216 |
| }, |
| "in_domain_correct": { |
| "value": 0.6792452830188679, |
| "ci95": [ |
| 0.626066051198163, |
| 0.7281456168580842 |
| ], |
| "n": 318, |
| "successes": 216 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.8836477987421384, |
| "ci95": [ |
| 0.843739488171426, |
| 0.9143977662647527 |
| ], |
| "n": 318, |
| "successes": 281 |
| }, |
| "by_group": { |
| "long": { |
| "value": 0.6981132075471698, |
| "ci95": [ |
| 0.5646316722203186, |
| 0.8048169734215012 |
| ], |
| "n": 53, |
| "successes": 37 |
| }, |
| "mixed": { |
| "value": 0.8571428571428571, |
| "ci95": [ |
| 0.486872167860775, |
| 0.9743203759421759 |
| ], |
| "n": 7, |
| "successes": 6 |
| }, |
| "plain": { |
| "value": 0.797979797979798, |
| "ci95": [ |
| 0.7084558880529249, |
| 0.8652427023968695 |
| ], |
| "n": 99, |
| "successes": 79 |
| }, |
| "slang": { |
| "value": 0.6037735849056604, |
| "ci95": [ |
| 0.4694147856894524, |
| 0.7241059334562627 |
| ], |
| "n": 53, |
| "successes": 32 |
| }, |
| "translit": { |
| "value": 0.3018867924528302, |
| "ci95": [ |
| 0.19518302657849879, |
| 0.4353683277796813 |
| ], |
| "n": 53, |
| "successes": 16 |
| }, |
| "typo": { |
| "value": 0.8679245283018868, |
| "ci95": [ |
| 0.751597445822794, |
| 0.9345214675120143 |
| ], |
| "n": 53, |
| "successes": 46 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.8301886792452831, |
| "ci95": [ |
| 0.7077471960878099, |
| 0.9080005466485566 |
| ], |
| "n": 53, |
| "successes": 44 |
| }, |
| "kk": { |
| "value": 0.7735849056603774, |
| "ci95": [ |
| 0.644737296378973, |
| 0.8654536904597305 |
| ], |
| "n": 53, |
| "successes": 41 |
| }, |
| "ru": { |
| "value": 0.6179245283018868, |
| "ci95": [ |
| 0.5509697085837892, |
| 0.680681802016396 |
| ], |
| "n": 212, |
| "successes": 131 |
| } |
| }, |
| "macro_f1_no_threshold": 0.887030394688552, |
| "per_intent": { |
| "about_university": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "academic_calendar": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "academic_integrity": { |
| "precision": 0.45454545454545453, |
| "recall": 0.8333333333333334, |
| "f1": 0.5882352941176471, |
| "support": 6 |
| }, |
| "academic_leave": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "academic_mobility": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "admission_deadlines": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "admission_documents": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "admission_steps": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "appeal": { |
| "precision": 0.625, |
| "recall": 0.8333333333333334, |
| "f1": 0.7142857142857143, |
| "support": 6 |
| }, |
| "capabilities": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "career_center": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "certificates": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "college_graduates": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "contacts": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "creative_exam": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "dormitory": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "ent_discounts": { |
| "precision": 0.625, |
| "recall": 0.8333333333333334, |
| "f1": 0.7142857142857143, |
| "support": 6 |
| }, |
| "ent_subjects": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "faculties": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "foreign_applicants": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "foundation": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "goodbye": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "grading_gpa": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "greeting": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "inclusive_education": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "international_exams": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "language_cert_discounts": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "library": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "location": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "master_admission": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "master_programs": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "medical_center": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "military": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "missed_exam": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "other_discounts": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "passing_score": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "payment_details": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "phd_admission": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "portals": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "practice": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "programs_list": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "retake_fx": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "schedule": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "state_grant": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "stipend": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "student_clubs": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "study_forms": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "thanks": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "transfer_restore": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "tuition_bachelor": { |
| "precision": 0.8, |
| "recall": 0.6666666666666666, |
| "f1": 0.7272727272727272, |
| "support": 6 |
| }, |
| "tuition_postgrad": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "turan_umiti_grant": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "vacant_grant": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| } |
| }, |
| "confusions": [ |
| { |
| "true": "greeting", |
| "pred": "goodbye", |
| "count": 1 |
| }, |
| { |
| "true": "capabilities", |
| "pred": "international_exams", |
| "count": 1 |
| }, |
| { |
| "true": "admission_steps", |
| "pred": "transfer_restore", |
| "count": 1 |
| }, |
| { |
| "true": "ent_subjects", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "passing_score", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "creative_exam", |
| "pred": "appeal", |
| "count": 1 |
| }, |
| { |
| "true": "college_graduates", |
| "pred": "appeal", |
| "count": 1 |
| }, |
| { |
| "true": "programs_list", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "study_forms", |
| "pred": "goodbye", |
| "count": 1 |
| }, |
| { |
| "true": "foreign_applicants", |
| "pred": "academic_integrity", |
| "count": 1 |
| } |
| ] |
| }, |
| "ood_test": { |
| "overall": { |
| "value": 0.7307692307692307, |
| "ci95": [ |
| 0.6232309971458905, |
| 0.8166438595843017 |
| ], |
| "n": 78, |
| "successes": 57 |
| }, |
| "ood_rejected": { |
| "value": 0.7307692307692307, |
| "ci95": [ |
| 0.6232309971458905, |
| 0.8166438595843017 |
| ], |
| "n": 78, |
| "successes": 57 |
| }, |
| "by_group": { |
| "near_domain": { |
| "value": 0.5652173913043478, |
| "ci95": [ |
| 0.3681137494517215, |
| 0.7436536478893558 |
| ], |
| "n": 23, |
| "successes": 13 |
| }, |
| "nonsense": { |
| "value": 0.7333333333333333, |
| "ci95": [ |
| 0.48049565748447975, |
| 0.8910254675038264 |
| ], |
| "n": 15, |
| "successes": 11 |
| }, |
| "offtopic": { |
| "value": 0.825, |
| "ci95": [ |
| 0.6805000953869541, |
| 0.9125458630150444 |
| ], |
| "n": 40, |
| "successes": 33 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.7777777777777778, |
| "ci95": [ |
| 0.5478541549399641, |
| 0.9099907195415445 |
| ], |
| "n": 18, |
| "successes": 14 |
| }, |
| "kk": { |
| "value": 0.8, |
| "ci95": [ |
| 0.4901624691221323, |
| 0.9433178490622331 |
| ], |
| "n": 10, |
| "successes": 8 |
| }, |
| "ru": { |
| "value": 0.7, |
| "ci95": [ |
| 0.5624964942316114, |
| 0.8089644656969429 |
| ], |
| "n": 50, |
| "successes": 35 |
| } |
| } |
| }, |
| "external": { |
| "overall": { |
| "value": 0.6363636363636364, |
| "ci95": [ |
| 0.5692297802032544, |
| 0.6985751873984265 |
| ], |
| "n": 209, |
| "successes": 133 |
| }, |
| "in_domain_correct": { |
| "value": 0.5214285714285715, |
| "ci95": [ |
| 0.4392192045992232, |
| 0.6024933867959423 |
| ], |
| "n": 140, |
| "successes": 73 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.75, |
| "ci95": [ |
| 0.6722461048304109, |
| 0.8144007947798533 |
| ], |
| "n": 140, |
| "successes": 105 |
| }, |
| "ood_rejected": { |
| "value": 0.8695652173913043, |
| "ci95": [ |
| 0.7703169996813442, |
| 0.9298337219442439 |
| ], |
| "n": 69, |
| "successes": 60 |
| }, |
| "by_group": { |
| "Caspian": { |
| "value": 0.6129032258064516, |
| "ci95": [ |
| 0.43824108487672686, |
| 0.7626689867666228 |
| ], |
| "n": 31, |
| "successes": 19 |
| }, |
| "EKTU": { |
| "value": 0.5714285714285714, |
| "ci95": [ |
| 0.25045836275303557, |
| 0.8417801460075546 |
| ], |
| "n": 7, |
| "successes": 4 |
| }, |
| "KIMEP": { |
| "value": 0.7857142857142857, |
| "ci95": [ |
| 0.5241076920169465, |
| 0.9242861334670647 |
| ], |
| "n": 14, |
| "successes": 11 |
| }, |
| "KarTU": { |
| "value": 0.3076923076923077, |
| "ci95": [ |
| 0.12680703567070326, |
| 0.5763065701492199 |
| ], |
| "n": 13, |
| "successes": 4 |
| }, |
| "NU": { |
| "value": 0.6666666666666666, |
| "ci95": [ |
| 0.29999331299474324, |
| 0.9032285897864654 |
| ], |
| "n": 6, |
| "successes": 4 |
| }, |
| "SDU": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "Satbayev": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "TorU": { |
| "value": 0.574468085106383, |
| "ci95": [ |
| 0.47353237545666105, |
| 0.6695562513900134 |
| ], |
| "n": 94, |
| "successes": 54 |
| }, |
| "Yasawi": { |
| "value": 0.78125, |
| "ci95": [ |
| 0.6124500620603575, |
| 0.8897616380720099 |
| ], |
| "n": 32, |
| "successes": 25 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.574468085106383, |
| "ci95": [ |
| 0.43281919237931477, |
| 0.7048637174746986 |
| ], |
| "n": 47, |
| "successes": 27 |
| }, |
| "kk": { |
| "value": 0.6071428571428571, |
| "ci95": [ |
| 0.42409044159980325, |
| 0.7643431355917387 |
| ], |
| "n": 28, |
| "successes": 17 |
| }, |
| "ru": { |
| "value": 0.664179104477612, |
| "ci95": [ |
| 0.58062954268048, |
| 0.738577757917602 |
| ], |
| "n": 134, |
| "successes": 89 |
| } |
| } |
| }, |
| "scenarios": { |
| "overall": { |
| "value": 0.8153846153846154, |
| "ci95": [ |
| 0.7044600606816986, |
| 0.8911112805749806 |
| ], |
| "n": 65, |
| "successes": 53 |
| }, |
| "in_domain_correct": { |
| "value": 0.8032786885245902, |
| "ci95": [ |
| 0.6869005932236627, |
| 0.8837219781736771 |
| ], |
| "n": 61, |
| "successes": 49 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.9016393442622951, |
| "ci95": [ |
| 0.8015539268574092, |
| 0.9541354244525815 |
| ], |
| "n": 61, |
| "successes": 55 |
| }, |
| "ood_rejected": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "by_group": { |
| "college_graduate": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "exchange_student": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "first_year_student": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "foreign_applicant": { |
| "value": 0.6666666666666666, |
| "ci95": [ |
| 0.29999331299474324, |
| 0.9032285897864654 |
| ], |
| "n": 6, |
| "successes": 4 |
| }, |
| "master_applicant": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "parent": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "phd_applicant": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| }, |
| "random_visitor": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "school_graduate": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "student_failed_exam": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5655175313406071, |
| 1.0 |
| ], |
| "n": 5, |
| "successes": 5 |
| }, |
| "student_paid_to_grant": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| }, |
| "student_special_needs": { |
| "value": 0.5, |
| "ci95": [ |
| 0.15003898774401986, |
| 0.8499610122559802 |
| ], |
| "n": 4, |
| "successes": 2 |
| }, |
| "student_with_family": { |
| "value": 0.5, |
| "ci95": [ |
| 0.15003898774401986, |
| 0.8499610122559802 |
| ], |
| "n": 4, |
| "successes": 2 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.7, |
| "ci95": [ |
| 0.3967781453459709, |
| 0.892208733443606 |
| ], |
| "n": 10, |
| "successes": 7 |
| }, |
| "kk": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "ru": { |
| "value": 0.8163265306122449, |
| "ci95": [ |
| 0.68642397281645, |
| 0.9002365898610293 |
| ], |
| "n": 49, |
| "successes": 40 |
| } |
| } |
| } |
| } |
| }, |
| "encoder/int8_emb_only": { |
| "size_mb": 173.2935733795166, |
| "session_rss_mb": 241.91015625, |
| "latency_ms": { |
| "p50": 13.482410999927197, |
| "p95": 22.6234977499871, |
| "p99": 31.01483033998193, |
| "mean": 14.93967196855504, |
| "n": 954 |
| }, |
| "top1_agreement_vs_fp32": 0.9968635650810246, |
| "cos_mean": 0.9999502301216125, |
| "cos_min": 0.9996614456176758, |
| "eval": { |
| "cv_accuracy": 0.8027444253859348, |
| "threshold": 0.731022679863181, |
| "test": { |
| "overall": { |
| "value": 0.6949685534591195, |
| "ci95": [ |
| 0.6422860107063452, |
| 0.7429968563022046 |
| ], |
| "n": 318, |
| "successes": 221 |
| }, |
| "in_domain_correct": { |
| "value": 0.6949685534591195, |
| "ci95": [ |
| 0.6422860107063452, |
| 0.7429968563022046 |
| ], |
| "n": 318, |
| "successes": 221 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.8805031446540881, |
| "ci95": [ |
| 0.8402335419115275, |
| 0.9116894727341905 |
| ], |
| "n": 318, |
| "successes": 280 |
| }, |
| "by_group": { |
| "long": { |
| "value": 0.6981132075471698, |
| "ci95": [ |
| 0.5646316722203186, |
| 0.8048169734215012 |
| ], |
| "n": 53, |
| "successes": 37 |
| }, |
| "mixed": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6456695613241317, |
| 1.0 |
| ], |
| "n": 7, |
| "successes": 7 |
| }, |
| "plain": { |
| "value": 0.8080808080808081, |
| "ci95": [ |
| 0.7195963930846633, |
| 0.8735496072108871 |
| ], |
| "n": 99, |
| "successes": 80 |
| }, |
| "slang": { |
| "value": 0.6415094339622641, |
| "ci95": [ |
| 0.5069114144879365, |
| 0.7569804752562204 |
| ], |
| "n": 53, |
| "successes": 34 |
| }, |
| "translit": { |
| "value": 0.32075471698113206, |
| "ci95": [ |
| 0.2109221400637985, |
| 0.4548147995936025 |
| ], |
| "n": 53, |
| "successes": 17 |
| }, |
| "typo": { |
| "value": 0.8679245283018868, |
| "ci95": [ |
| 0.751597445822794, |
| 0.9345214675120143 |
| ], |
| "n": 53, |
| "successes": 46 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.8679245283018868, |
| "ci95": [ |
| 0.751597445822794, |
| 0.9345214675120143 |
| ], |
| "n": 53, |
| "successes": 46 |
| }, |
| "kk": { |
| "value": 0.7735849056603774, |
| "ci95": [ |
| 0.644737296378973, |
| 0.8654536904597305 |
| ], |
| "n": 53, |
| "successes": 41 |
| }, |
| "ru": { |
| "value": 0.6320754716981132, |
| "ci95": [ |
| 0.5653473327892489, |
| 0.6941023590829587 |
| ], |
| "n": 212, |
| "successes": 134 |
| } |
| }, |
| "macro_f1_no_threshold": 0.8842913098739955, |
| "per_intent": { |
| "about_university": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "academic_calendar": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "academic_integrity": { |
| "precision": 0.45454545454545453, |
| "recall": 0.8333333333333334, |
| "f1": 0.5882352941176471, |
| "support": 6 |
| }, |
| "academic_leave": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "academic_mobility": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "admission_deadlines": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "admission_documents": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "support": 6 |
| }, |
| "admission_steps": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "appeal": { |
| "precision": 0.6666666666666666, |
| "recall": 1.0, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "capabilities": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "career_center": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "certificates": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "college_graduates": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "contacts": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "creative_exam": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "dormitory": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "ent_discounts": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "ent_subjects": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "faculties": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "foreign_applicants": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "foundation": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "goodbye": { |
| "precision": 0.6, |
| "recall": 1.0, |
| "f1": 0.7499999999999999, |
| "support": 6 |
| }, |
| "grading_gpa": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "greeting": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "inclusive_education": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "international_exams": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "language_cert_discounts": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "library": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "location": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "master_admission": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "master_programs": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "medical_center": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "military": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "missed_exam": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "other_discounts": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "passing_score": { |
| "precision": 0.7142857142857143, |
| "recall": 0.8333333333333334, |
| "f1": 0.7692307692307692, |
| "support": 6 |
| }, |
| "payment_details": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "phd_admission": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "portals": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "practice": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "programs_list": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "retake_fx": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "schedule": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "state_grant": { |
| "precision": 1.0, |
| "recall": 0.6666666666666666, |
| "f1": 0.8, |
| "support": 6 |
| }, |
| "stipend": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "student_clubs": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "study_forms": { |
| "precision": 0.8333333333333334, |
| "recall": 0.8333333333333334, |
| "f1": 0.8333333333333334, |
| "support": 6 |
| }, |
| "thanks": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| }, |
| "transfer_restore": { |
| "precision": 0.75, |
| "recall": 1.0, |
| "f1": 0.8571428571428571, |
| "support": 6 |
| }, |
| "tuition_bachelor": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "tuition_postgrad": { |
| "precision": 1.0, |
| "recall": 0.8333333333333334, |
| "f1": 0.9090909090909091, |
| "support": 6 |
| }, |
| "turan_umiti_grant": { |
| "precision": 0.8571428571428571, |
| "recall": 1.0, |
| "f1": 0.923076923076923, |
| "support": 6 |
| }, |
| "vacant_grant": { |
| "precision": 1.0, |
| "recall": 1.0, |
| "f1": 1.0, |
| "support": 6 |
| } |
| }, |
| "confusions": [ |
| { |
| "true": "greeting", |
| "pred": "goodbye", |
| "count": 1 |
| }, |
| { |
| "true": "capabilities", |
| "pred": "international_exams", |
| "count": 1 |
| }, |
| { |
| "true": "admission_steps", |
| "pred": "transfer_restore", |
| "count": 1 |
| }, |
| { |
| "true": "ent_subjects", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "passing_score", |
| "pred": "ent_discounts", |
| "count": 1 |
| }, |
| { |
| "true": "creative_exam", |
| "pred": "appeal", |
| "count": 1 |
| }, |
| { |
| "true": "college_graduates", |
| "pred": "missed_exam", |
| "count": 1 |
| }, |
| { |
| "true": "programs_list", |
| "pred": "medical_center", |
| "count": 1 |
| }, |
| { |
| "true": "programs_list", |
| "pred": "transfer_restore", |
| "count": 1 |
| }, |
| { |
| "true": "study_forms", |
| "pred": "goodbye", |
| "count": 1 |
| } |
| ] |
| }, |
| "ood_test": { |
| "overall": { |
| "value": 0.7692307692307693, |
| "ci95": [ |
| 0.6644424208413748, |
| 0.8487449120105162 |
| ], |
| "n": 78, |
| "successes": 60 |
| }, |
| "ood_rejected": { |
| "value": 0.7692307692307693, |
| "ci95": [ |
| 0.6644424208413748, |
| 0.8487449120105162 |
| ], |
| "n": 78, |
| "successes": 60 |
| }, |
| "by_group": { |
| "near_domain": { |
| "value": 0.5652173913043478, |
| "ci95": [ |
| 0.3681137494517215, |
| 0.7436536478893558 |
| ], |
| "n": 23, |
| "successes": 13 |
| }, |
| "nonsense": { |
| "value": 0.8666666666666667, |
| "ci95": [ |
| 0.6211801694601944, |
| 0.9626387412357151 |
| ], |
| "n": 15, |
| "successes": 13 |
| }, |
| "offtopic": { |
| "value": 0.85, |
| "ci95": [ |
| 0.7092767550401498, |
| 0.9293881232389256 |
| ], |
| "n": 40, |
| "successes": 34 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.7777777777777778, |
| "ci95": [ |
| 0.5478541549399641, |
| 0.9099907195415445 |
| ], |
| "n": 18, |
| "successes": 14 |
| }, |
| "kk": { |
| "value": 0.8, |
| "ci95": [ |
| 0.4901624691221323, |
| 0.9433178490622331 |
| ], |
| "n": 10, |
| "successes": 8 |
| }, |
| "ru": { |
| "value": 0.76, |
| "ci95": [ |
| 0.6258731612803865, |
| 0.8570260866267341 |
| ], |
| "n": 50, |
| "successes": 38 |
| } |
| } |
| }, |
| "external": { |
| "overall": { |
| "value": 0.6267942583732058, |
| "ci95": [ |
| 0.5594889825720564, |
| 0.6895226539698576 |
| ], |
| "n": 209, |
| "successes": 131 |
| }, |
| "in_domain_correct": { |
| "value": 0.5142857142857142, |
| "ci95": [ |
| 0.43222650430712206, |
| 0.5955818899563214 |
| ], |
| "n": 140, |
| "successes": 72 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.7642857142857142, |
| "ci95": [ |
| 0.6875066304583293, |
| 0.8269486634153786 |
| ], |
| "n": 140, |
| "successes": 107 |
| }, |
| "ood_rejected": { |
| "value": 0.855072463768116, |
| "ci95": [ |
| 0.7533647830164287, |
| 0.9193290475650187 |
| ], |
| "n": 69, |
| "successes": 59 |
| }, |
| "by_group": { |
| "Caspian": { |
| "value": 0.6129032258064516, |
| "ci95": [ |
| 0.43824108487672686, |
| 0.7626689867666228 |
| ], |
| "n": 31, |
| "successes": 19 |
| }, |
| "EKTU": { |
| "value": 0.5714285714285714, |
| "ci95": [ |
| 0.25045836275303557, |
| 0.8417801460075546 |
| ], |
| "n": 7, |
| "successes": 4 |
| }, |
| "KIMEP": { |
| "value": 0.7857142857142857, |
| "ci95": [ |
| 0.5241076920169465, |
| 0.9242861334670647 |
| ], |
| "n": 14, |
| "successes": 11 |
| }, |
| "KarTU": { |
| "value": 0.3076923076923077, |
| "ci95": [ |
| 0.12680703567070326, |
| 0.5763065701492199 |
| ], |
| "n": 13, |
| "successes": 4 |
| }, |
| "NU": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "SDU": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "Satbayev": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "TorU": { |
| "value": 0.5851063829787234, |
| "ci95": [ |
| 0.4840802044979575, |
| 0.6794496547553848 |
| ], |
| "n": 94, |
| "successes": 55 |
| }, |
| "Yasawi": { |
| "value": 0.71875, |
| "ci95": [ |
| 0.5462549043349592, |
| 0.8443541957679932 |
| ], |
| "n": 32, |
| "successes": 23 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.5319148936170213, |
| "ci95": [ |
| 0.39232383116617336, |
| 0.6666831301998322 |
| ], |
| "n": 47, |
| "successes": 25 |
| }, |
| "kk": { |
| "value": 0.6071428571428571, |
| "ci95": [ |
| 0.42409044159980325, |
| 0.7643431355917387 |
| ], |
| "n": 28, |
| "successes": 17 |
| }, |
| "ru": { |
| "value": 0.664179104477612, |
| "ci95": [ |
| 0.58062954268048, |
| 0.738577757917602 |
| ], |
| "n": 134, |
| "successes": 89 |
| } |
| } |
| }, |
| "scenarios": { |
| "overall": { |
| "value": 0.8307692307692308, |
| "ci95": [ |
| 0.7218357826609499, |
| 0.9027878191448359 |
| ], |
| "n": 65, |
| "successes": 54 |
| }, |
| "in_domain_correct": { |
| "value": 0.819672131147541, |
| "ci95": [ |
| 0.7052584355013631, |
| 0.8962085992147519 |
| ], |
| "n": 61, |
| "successes": 50 |
| }, |
| "in_domain_top1_no_threshold": { |
| "value": 0.9344262295081968, |
| "ci95": [ |
| 0.8431719822578765, |
| 0.9742062956896644 |
| ], |
| "n": 61, |
| "successes": 57 |
| }, |
| "ood_rejected": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "by_group": { |
| "college_graduate": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "exchange_student": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "first_year_student": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "foreign_applicant": { |
| "value": 0.6666666666666666, |
| "ci95": [ |
| 0.29999331299474324, |
| 0.9032285897864654 |
| ], |
| "n": 6, |
| "successes": 4 |
| }, |
| "master_applicant": { |
| "value": 0.8, |
| "ci95": [ |
| 0.375534627029995, |
| 0.9637758917743693 |
| ], |
| "n": 5, |
| "successes": 4 |
| }, |
| "parent": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "phd_applicant": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5101091596030786, |
| 0.9999999999999999 |
| ], |
| "n": 4, |
| "successes": 4 |
| }, |
| "random_visitor": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "school_graduate": { |
| "value": 0.8333333333333334, |
| "ci95": [ |
| 0.4364971749742326, |
| 0.969946630588185 |
| ], |
| "n": 6, |
| "successes": 5 |
| }, |
| "student_failed_exam": { |
| "value": 1.0, |
| "ci95": [ |
| 0.5655175313406071, |
| 1.0 |
| ], |
| "n": 5, |
| "successes": 5 |
| }, |
| "student_paid_to_grant": { |
| "value": 0.75, |
| "ci95": [ |
| 0.3006418400969081, |
| 0.954412739704631 |
| ], |
| "n": 4, |
| "successes": 3 |
| }, |
| "student_special_needs": { |
| "value": 0.5, |
| "ci95": [ |
| 0.15003898774401986, |
| 0.8499610122559802 |
| ], |
| "n": 4, |
| "successes": 2 |
| }, |
| "student_with_family": { |
| "value": 0.5, |
| "ci95": [ |
| 0.15003898774401986, |
| 0.8499610122559802 |
| ], |
| "n": 4, |
| "successes": 2 |
| } |
| }, |
| "by_lang": { |
| "en": { |
| "value": 0.8, |
| "ci95": [ |
| 0.4901624691221323, |
| 0.9433178490622331 |
| ], |
| "n": 10, |
| "successes": 8 |
| }, |
| "kk": { |
| "value": 1.0, |
| "ci95": [ |
| 0.6096657083436265, |
| 1.0 |
| ], |
| "n": 6, |
| "successes": 6 |
| }, |
| "ru": { |
| "value": 0.8163265306122449, |
| "ci95": [ |
| 0.68642397281645, |
| 0.9002365898610293 |
| ], |
| "n": 49, |
| "successes": 40 |
| } |
| } |
| } |
| } |
| } |
| }, |
| "intents": [ |
| "about_university", |
| "academic_calendar", |
| "academic_integrity", |
| "academic_leave", |
| "academic_mobility", |
| "admission_deadlines", |
| "admission_documents", |
| "admission_steps", |
| "appeal", |
| "capabilities", |
| "career_center", |
| "certificates", |
| "college_graduates", |
| "contacts", |
| "creative_exam", |
| "dormitory", |
| "ent_discounts", |
| "ent_subjects", |
| "faculties", |
| "foreign_applicants", |
| "foundation", |
| "goodbye", |
| "grading_gpa", |
| "greeting", |
| "inclusive_education", |
| "international_exams", |
| "language_cert_discounts", |
| "library", |
| "location", |
| "master_admission", |
| "master_programs", |
| "medical_center", |
| "military", |
| "missed_exam", |
| "other_discounts", |
| "passing_score", |
| "payment_details", |
| "phd_admission", |
| "portals", |
| "practice", |
| "programs_list", |
| "retake_fx", |
| "schedule", |
| "state_grant", |
| "stipend", |
| "student_clubs", |
| "study_forms", |
| "thanks", |
| "transfer_restore", |
| "tuition_bachelor", |
| "tuition_postgrad", |
| "turan_umiti_grant", |
| "vacant_grant" |
| ], |
| "frozen_C": 1000, |
| "settings": { |
| "max_length": 64, |
| "query_prefix": "query: ", |
| "ft_epochs": 15, |
| "ft_lr_encoder": 2e-05, |
| "ft_lr_head": 0.001, |
| "seed": 42 |
| } |
| } |
