import{a as I,f as te,c as Q_,t as N_,d as tt}from"../chunks/D0-Lpew9.js";import"../chunks/DP5BbXZE.js";import{o as B_}from"../chunks/Cx7tSjKg.js";import{c as We,g as n_,aI as i_,h as se,l as Qe,F as a_,i as F_,L as e,j as D_,S as K_,k as Dt,n as Ne,f as Le,ab as s_,aj as $_,a0 as Kt,a as H_,aJ as xe,b as et,aK as Y_,s as U_,aC as fe,aL as G_,ai as wt,aM as V_,aN as X_,ao as ee,P as $t,aO as W_,ax as eo,r as l_,p as r_,aP as gt,O as c_,aQ as to,aR as _o,ag as oo,m as no,d as io,ad as ft,D as ao,E as so,aS as lo,R as ro,ap as m_,t as c,aT as co,aU as mo,aV as Ie,a8 as d_,aW as po,aX as Ht,aY as Yt,aZ as uo,a_ as go,a$ as ho,b0 as bo,b1 as fo,b2 as yo,q as vo,aE as wo,u as Ut,b3 as Gt,b4 as xo,b5 as C,aB as ko,v as Po,al as Y,b6 as ze,b7 as jo,x as V,y as qo,z as h,B as x,A as g,a1 as ht,b8 as Co,w as Ue}from"../chunks/CdZ30l9o.js";import{a as To,d as Eo,b as he,s as N,e as Ge}from"../chunks/CryQBVor.js";import{i as be}from"../chunks/DHQwSDIs.js";function Oe(t,_){return _}function So(t,_,s){for(var r=[],d=_.length,l,a=_.length,u=0;u<d;u++){let q=_[u];r_(q,()=>{if(l){if(l.pending.delete(q),l.done.add(q),l.pending.size===0){var S=t.outrogroups;yt(t,wt(l.done)),S.delete(l),S.size===0&&(t.outrogroups=null)}}else a-=1},!1)}if(a===0){var v=r.length===0&&s!==null;if(v){var b=s,f=b.parentNode;oo(f),f.append(b),t.items.clear()}yt(t,_,!v)}else l={pending:new Set(_),done:new Set},(t.outrogroups??(t.outrogroups=new Set)).add(l)}function yt(t,_,s=!0){var r;if(t.pending.size>0){r=new Set;for(const a of t.pending.values())for(const u of a)r.add(t.items.get(u).e)}for(var d=0;d<_.length;d++){var l=_[d];if(r!=null&&r.has(l)){l.f|=xe;const a=document.createDocumentFragment();no(l,a)}else io(_[d],s)}}var Vt;function we(t,_,s,r,d,l=null){var a=t,u=new Map,v=(_&i_)!==0;if(v){var b=t;a=se?Qe(a_(b)):b.appendChild(We())}se&&F_();var f=null,q=fe(()=>{var M=s();return G_(M)?M:M==null?[]:wt(M)}),S,O=new Map,R=!0;function Z(M){(F.effect.f&eo)===0&&(F.pending.delete(M),F.fallback=f,Ro(F,S,a,_,r),f!==null&&(S.length===0?(f.f&xe)===0?l_(f):(f.f^=xe,Ze(f,null,a)):r_(f,()=>{f=null})))}function m(M){F.pending.delete(M)}var y=n_(()=>{S=e(q);var M=S.length;let U=!1;if(se){var _e=D_(a)===K_;_e!==(M===0)&&(a=Dt(),Qe(a),Ne(!1),U=!0)}for(var W=new Set,$=H_,pe=U_(),le=0;le<M;le+=1){se&&Le.nodeType===s_&&Le.data===$_&&(a=Le,U=!0,Ne(!1));var re=S[le],ke=r(re,le),ne=R?null:u.get(ke);ne?(ne.v&&Kt(ne.v,re),ne.i&&Kt(ne.i,le),pe&&$.unskip_effect(ne.e)):(ne=zo(u,R?a:Vt??(Vt=We()),re,ke,le,d,_,s),R||(ne.e.f|=xe),u.set(ke,ne)),W.add(ke)}if(M===0&&l&&!f&&(R?f=et(()=>l(a)):(f=et(()=>l(Vt??(Vt=We()))),f.f|=xe)),M>W.size&&Y_(),se&&M>0&&Qe(Dt()),!R)if(O.set($,W),pe){for(const[ye,X]of u)W.has(ye)||$.skip_effect(X.e);$.oncommit(Z),$.ondiscard(m)}else Z($);U&&Ne(!0),e(q)}),F={effect:y,items:u,pending:O,outrogroups:null,fallback:f};R=!1,se&&(a=Le)}function Me(t){for(;t!==null&&(t.f&to)===0;)t=t.next;return t}function Ro(t,_,s,r,d){var re,ke,ne,ye,X,Te,Be,Ae,Re;var l=(r&_o)!==0,a=_.length,u=t.items,v=Me(t.effect.first),b,f=null,q,S=[],O=[],R,Z,m,y;if(l)for(y=0;y<a;y+=1)R=_[y],Z=d(R,y),m=u.get(Z).e,(m.f&xe)===0&&((ke=(re=m.nodes)==null?void 0:re.a)==null||ke.measure(),(q??(q=new Set)).add(m));for(y=0;y<a;y+=1){if(R=_[y],Z=d(R,y),m=u.get(Z).e,t.outrogroups!==null)for(const ue of t.outrogroups)ue.pending.delete(m),ue.done.delete(m);if((m.f&gt)!==0&&(l_(m),l&&((ye=(ne=m.nodes)==null?void 0:ne.a)==null||ye.unfix(),(q??(q=new Set)).delete(m))),(m.f&xe)!==0)if(m.f^=xe,m===v)Ze(m,null,s);else{var F=f?f.next:v;m===t.effect.last&&(t.effect.last=m.prev),m.prev&&(m.prev.next=m.next),m.next&&(m.next.prev=m.prev),Se(t,f,m),Se(t,m,F),Ze(m,F,s),f=m,S=[],O=[],v=Me(f.next);continue}if(m!==v){if(b!==void 0&&b.has(m)){if(S.length<O.length){var M=O[0],U;f=M.prev;var _e=S[0],W=S[S.length-1];for(U=0;U<S.length;U+=1)Ze(S[U],M,s);for(U=0;U<O.length;U+=1)b.delete(O[U]);Se(t,_e.prev,W.next),Se(t,f,_e),Se(t,W,M),v=M,f=W,y-=1,S=[],O=[]}else b.delete(m),Ze(m,v,s),Se(t,m.prev,m.next),Se(t,m,f===null?t.effect.first:f.next),Se(t,f,m),f=m;continue}for(S=[],O=[];v!==null&&v!==m;)(b??(b=new Set)).add(v),O.push(v),v=Me(v.next);if(v===null)continue}(m.f&xe)===0&&S.push(m),f=m,v=Me(m.next)}if(t.outrogroups!==null){for(const ue of t.outrogroups)ue.pending.size===0&&(yt(t,wt(ue.done)),(X=t.outrogroups)==null||X.delete(ue));t.outrogroups.size===0&&(t.outrogroups=null)}if(v!==null||b!==void 0){var $=[];if(b!==void 0)for(m of b)(m.f&gt)===0&&$.push(m);for(;v!==null;)(v.f&gt)===0&&v!==t.fallback&&$.push(v),v=Me(v.next);var pe=$.length;if(pe>0){var le=(r&i_)!==0&&a===0?s:null;if(l){for(y=0;y<pe;y+=1)(Be=(Te=$[y].nodes)==null?void 0:Te.a)==null||Be.measure();for(y=0;y<pe;y+=1)(Re=(Ae=$[y].nodes)==null?void 0:Ae.a)==null||Re.fix()}So(t,$,le)}}l&&c_(()=>{var ue,Fe;if(q!==void 0)for(m of q)(Fe=(ue=m.nodes)==null?void 0:ue.a)==null||Fe.apply()})}function zo(t,_,s,r,d,l,a,u){var v=(a&V_)!==0?(a&X_)===0?ee(s,!1,!1):$t(s):null,b=(a&W_)!==0?$t(d):null;return{v,i:b,e:et(()=>(l(_,v??s,b??d,u),()=>{t.delete(r)}))}}function Ze(t,_,s){if(t.nodes)for(var r=t.nodes.start,d=t.nodes.end,l=_&&(_.f&xe)===0?_.nodes.start:s;r!==null;){var a=ft(r);if(l.before(r),r===d)return;r=a}}function Se(t,_,s){_===null?t.effect.first=s:_.next=s,s===null?t.effect.last=_:s.prev=_}const Oo=()=>performance.now(),Ce={tick:t=>requestAnimationFrame(t),now:()=>Oo(),tasks:new Set};function p_(){const t=Ce.now();Ce.tasks.forEach(_=>{_.c(t)||(Ce.tasks.delete(_),_.f())}),Ce.tasks.size!==0&&Ce.tick(p_)}function Jo(t){let _;return Ce.tasks.size===0&&Ce.tick(p_),{promise:new Promise(s=>{Ce.tasks.add(_={c:t,f:s})}),abort(){Ce.tasks.delete(_)}}}function Ve(t,_){d_(()=>{t.dispatchEvent(new CustomEvent(_))})}function Ao(t){if(t==="float")return"cssFloat";if(t==="offset")return"cssOffset";if(t.startsWith("--"))return t;const _=t.split("-");return _.length===1?_[0]:_[0]+_.slice(1).map(s=>s[0].toUpperCase()+s.slice(1)).join("")}function Xt(t){const _={},s=t.split(";");for(const r of s){const[d,l]=r.split(":");if(!d||l===void 0)break;const a=Ao(d.trim());_[a]=l.trim()}return _}const Io=t=>t;function Wt(t,_,s,r){var m;var d=(t&co)!==0,l="both",a,u=_.inert,v=_.style.overflow,b,f;function q(){return d_(()=>a??(a=s()(_,(r==null?void 0:r())??{},{direction:l})))}var S={is_global:d,in(){_.inert=u,b=vt(_,q(),f,1,()=>{Ve(_,"introstart")},()=>{Ve(_,"introend"),b==null||b.abort(),b=a=void 0,_.style.overflow=v})},out(y){_.inert=!0,f=vt(_,q(),b,0,()=>{Ve(_,"outrostart")},()=>{Ve(_,"outroend"),y==null||y()})},stop:()=>{b==null||b.abort(),f==null||f.abort()}},O=ao;if(((m=O.nodes).t??(m.t=[])).push(S),To){var R=d;if(!R){for(var Z=O.parent;Z&&(Z.f&so)!==0;)for(;(Z=Z.parent)&&(Z.f&lo)===0;);R=!Z||(Z.f&ro)!==0}R&&m_(()=>{c(()=>S.in())})}}function vt(t,_,s,r,d,l){var a=r===1;if(mo(_)){var u,v=!1;return c_(()=>{if(!v){var y=_({direction:a?"in":"out"});u=vt(t,y,s,r,d,l)}}),{abort:()=>{v=!0,u==null||u.abort()},deactivate:()=>u.deactivate(),reset:()=>u.reset(),t:()=>u.t()}}if(s==null||s.deactivate(),!(_!=null&&_.duration)&&!(_!=null&&_.delay))return d(),l(),{abort:Ie,deactivate:Ie,reset:Ie,t:()=>r};const{delay:b=0,css:f,tick:q,easing:S=Io}=_;var O=[];if(a&&s===void 0&&(q&&q(0,1),f)){var R=Xt(f(0,1));O.push(R,R)}var Z=()=>1-r,m=t.animate(O,{duration:b,fill:"forwards"});return m.onfinish=()=>{m.cancel(),d();var y=(s==null?void 0:s.t())??1-r;s==null||s.abort();var F=r-y,M=_.duration*Math.abs(F),U=[];if(M>0){var _e=!1;if(f)for(var W=Math.ceil(M/16.666666666666668),$=0;$<=W;$+=1){var pe=y+F*S($/W),le=Xt(f(pe,1-pe));U.push(le),_e||(_e=le.overflow==="hidden")}_e&&(t.style.overflow="hidden"),Z=()=>{var re=m.currentTime;return y+F*S(re/M)},q&&Jo(()=>{if(m.playState!=="running")return!1;var re=Z();return q(re,1-re),!0})}m=t.animate(U,{duration:M,fill:"forwards"}),m.onfinish=()=>{Z=()=>r,q==null||q(r,1-r),l()}},{abort:()=>{m&&(m.cancel(),m.effect=null,m.onfinish=Ie)},deactivate:()=>{l=Ie},reset:()=>{r===0&&(q==null||q(1,0))},t:()=>Z()}}function Mo(t,_){let s=null,r=se;var d;if(se){s=Le;for(var l=a_(document.head);l!==null&&(l.nodeType!==s_||l.data!==t);)l=ft(l);if(l===null)Ne(!1);else{var a=ft(l);l.remove(),Qe(a)}}se||(d=document.head.appendChild(We()));try{n_(()=>{var u=et(()=>_(d));u.f|=po})}finally{r&&(Ne(!0),Qe(s))}}const e_=[...` 	
\r\f \v\uFEFF`];function Lo(t,_,s){var r=t==null?"":""+t;if(s){for(var d of Object.keys(s))if(s[d])r=r?r+" "+d:d;else if(r.length)for(var l=d.length,a=0;(a=r.indexOf(d,a))>=0;){var u=a+l;(a===0||e_.includes(r[a-1]))&&(u===r.length||e_.includes(r[u]))?r=(a===0?"":r.substring(0,a))+r.substring(u+1):a=u}}return r===""?null:r}function Zo(t,_){return t==null?null:String(t)}function oe(t,_,s,r,d,l){var a=t[Ht];if(se||a!==s||a===void 0){var u=Lo(s,r,l);(!se||u!==t.getAttribute("class"))&&(u==null?t.removeAttribute("class"):_?t.className=u:t.setAttribute("class",u)),t[Ht]=s}else if(l&&d!==l)for(var v in l){var b=!!l[v];(d==null||b!==!!d[v])&&t.classList.toggle(v,b)}return l}function Xe(t,_,s,r){var d=t[Yt];if(se||d!==_){var l=Zo(_);(!se||l!==t.getAttribute("style"))&&(l==null?t.removeAttribute("style"):t.style.cssText=l),t[Yt]=_}return r}const Qo=Symbol("is custom element"),No=Symbol("is html"),Bo=yo?"link":"LINK";function E(t,_,s,r){var d=Fo(t);se&&(d[_]=t.getAttribute(_),_==="src"||_==="srcset"||_==="href"&&t.nodeName===Bo)||d[_]!==(d[_]=s)&&(_==="loading"&&(t[fo]=s),s==null?t.removeAttribute(_):typeof s!="string"&&Do(t).includes(_)?t[_]=s:t.setAttribute(_,s))}function Fo(t){var _;return t[_=uo]??(t[_]={[Qo]:t.nodeName.includes("-"),[No]:t.namespaceURI===go})}var t_=new Map;function Do(t){var _=t.getAttribute("is")||t.nodeName,s=t_.get(_);if(s)return s;t_.set(_,s=[]);for(var r,d=t,l=Element.prototype;l!==d;){r=bo(d);for(var a in r)r[a].set&&a!=="innerHTML"&&a!=="textContent"&&a!=="innerText"&&s.push(a);d=ho(d)}return s}function Ko(t=!1){const _=vo,s=_.l.u;if(!s)return;let r=()=>C(_.s);if(t){let d=0,l={};const a=ko(()=>{let u=!1;const v=_.s;for(const b in v)v[b]!==l[b]&&(l[b]=v[b],u=!0);return u&&d++,d});r=()=>e(a)}s.b.length&&wo(()=>{__(_,r),Gt(s.b)}),Ut(()=>{const d=c(()=>s.m.map(xo));return()=>{for(const l of d)typeof l=="function"&&l()}}),s.a.length&&Ut(()=>{__(_,r),Gt(s.a)})}function __(t,_){if(t.l.s)for(const s of t.l.s)e(s);_()}const $o=t=>t;function Ho(t){const _=t-1;return _*_*_+1}function Yo(t,{delay:_=0,duration:s=400,easing:r=$o}={}){const d=+getComputedStyle(t).opacity;return{delay:_,duration:s,easing:r,css:l=>`opacity: ${l*d}`}}function Uo(t,{delay:_=0,duration:s=400,easing:r=Ho,axis:d="y"}={}){const l=getComputedStyle(t),a=+l.opacity,u=d==="y"?"height":"width",v=parseFloat(l[u]),b=d==="y"?["top","bottom"]:["left","right"],f=b.map(y=>`${y[0].toUpperCase()}${y.slice(1)}`),q=parseFloat(l[`padding${f[0]}`]),S=parseFloat(l[`padding${f[1]}`]),O=parseFloat(l[`margin${f[0]}`]),R=parseFloat(l[`margin${f[1]}`]),Z=parseFloat(l[`border${f[0]}Width`]),m=parseFloat(l[`border${f[1]}Width`]);return{delay:_,duration:s,easing:r,css:y=>`overflow: hidden;opacity: ${Math.min(y*20,1)*a};${u}: ${y*v}px;padding-${b[0]}: ${y*q}px;padding-${b[1]}: ${y*S}px;margin-${b[0]}: ${y*O}px;margin-${b[1]}: ${y*R}px;border-${b[0]}-width: ${y*Z}px;border-${b[1]}-width: ${y*m}px;min-${u}: 0`}}const Go=JSON.parse('{"atom-3-4m__gpt-s-1-4m":{"ci":[1.0725360001654316,2.9948782002109477],"difference":2.0326431301418864,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__atom-3-4m":{"ci":[14.14042731298988,16.348961152329448],"difference":15.263255663614302,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__falcon-h1-tiny-r-90m":{"ci":[7.754579096585064,9.903861793761253],"difference":8.820217466378272,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__gpt-s-1-4m":{"ci":[16.129475466747913,18.474038330502612],"difference":17.295898793756187,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__gpt-s2-5m":{"ci":[9.256397771070153,11.38004880060208],"difference":10.331448058039745,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__gpt2-124m":{"ci":[0.7505292033322868,2.7143772000682045],"difference":1.7402333545016724,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__nanowhale-100m-base":{"ci":[10.704834399063397,12.75788043869504],"difference":11.742736810760391,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__pythia-160m":{"ci":[8.298184537762872,10.43661712641525],"difference":9.394030404387307,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__pythia-31m":{"ci":[11.57570162659012,13.766851356448733],"difference":12.659259145773355,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__slm-10m":{"ci":[8.323210002923028,10.392029505159467],"difference":9.360557666936613,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__supra-50m-base":{"ci":[1.3509633161095662,3.2857258513374914],"difference":2.3278594898349447,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__veyra2-apricot-50m-base":{"ci":[2.5296915340167394,4.5001965180924905],"difference":3.547305330072205,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__atom-3-4m":{"ci":[5.420215532883351,7.444210689454811],"difference":6.44303819723603,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__gpt-s-1-4m":{"ci":[7.39930477778452,9.52758940136927],"difference":8.475681327377918,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__gpt-s2-5m":{"ci":[0.5031066010624538,2.4698466105128163],"difference":1.5112305916614743,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__nanowhale-100m-base":{"ci":[1.8741543839787467,3.9548193375016245],"difference":2.922519344382119,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__pythia-160m":{"ci":[-0.580794542209572,1.6693586430525291],"difference":0.5738129380090358,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"falcon-h1-tiny-r-90m__pythia-31m":{"ci":[2.730495758942819,4.922286953231492],"difference":3.8390416793950854,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__slm-10m":{"ci":[-0.4400500092347234,1.5559306125986496],"difference":0.5403402005583414,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"gemma-3-270m__atom-3-4m":{"ci":[23.385199210048242,25.661827101907907],"difference":24.542681336414624,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__baguettotron":{"ci":[8.231872620662342,10.330917113466759],"difference":9.279425672800324,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__falcon-h1-tiny-r-90m":{"ci":[16.984766782761056,19.19024856397875],"difference":18.099643139178596,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt-s-1-4m":{"ci":[25.31383386993303,27.774047690234024],"difference":26.57532446655651,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt-s2-5m":{"ci":[18.451728069080684,20.73233091916342],"difference":19.61087373084007,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt-x2-125m":{"ci":[2.5100180469723314,4.277907176531071],"difference":3.395722404973594,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt2-124m":{"ci":[10.033805987347693,12.02164981680341],"difference":11.019659027301998,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__mobilellm-r1-140m-base":{"ci":[6.7285212269252535,8.654045161821886],"difference":7.708492950574182,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__nanowhale-100m-base":{"ci":[19.911079186076847,22.136911037973743],"difference":21.022162483560713,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__pythia-160m":{"ci":[17.49071148889552,19.77861550252012],"difference":18.67345607718763,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__pythia-31m":{"ci":[20.72644303179628,23.094161479902546],"difference":21.93868481857368,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__slm-10m":{"ci":[17.546740227831336,19.736908231495857],"difference":18.639983339736936,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__smollm2-135m":{"ci":[0.16836524011662418,1.9651071736191295],"difference":1.0576894080699517,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__supra-50m-base":{"ci":[10.628267365916303,12.551452199235303],"difference":11.60728516263527,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__veyra2-apricot-50m-base":{"ci":[11.848671246541473,13.81413489824284],"difference":12.82673100287253,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__atom-3-4m":{"ci":[4.048447096687848,5.816152073030026],"difference":4.931807605574556,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__gpt-s-1-4m":{"ci":[5.994355598920947,7.87200612832699],"difference":6.964450735716443,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__nanowhale-100m-base":{"ci":[0.48907344006903475,2.3424471779865126],"difference":1.4112887527206444,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__pythia-31m":{"ci":[1.3601371630505645,3.270819706973676],"difference":2.3278110877336107,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__atom-3-4m":{"ci":[20.055298011374177,22.273700065241986],"difference":21.146958931441034,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__baguettotron":{"ci":[4.919037209936775,6.853051793908139],"difference":5.883703267826731,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__falcon-h1-tiny-r-90m":{"ci":[13.662735239486574,15.718124447804271],"difference":14.703920734205003,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__gpt-s-1-4m":{"ci":[22.00694789823436,24.304170912361922],"difference":23.17960206158292,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__gpt-s2-5m":{"ci":[15.169372508200368,17.236224767654043],"difference":16.215151325866476,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__gpt2-124m":{"ci":[6.747015411597489,8.522372794247302],"difference":7.623936622328404,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__mobilellm-r1-140m-base":{"ci":[3.4001537890801874,5.2268186213654895],"difference":4.31277054560059,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__nanowhale-100m-base":{"ci":[16.613503795581103,18.6676187860639],"difference":17.626440078587123,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__pythia-160m":{"ci":[14.206915778777304,16.37469514048295],"difference":15.277733672214039,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__pythia-31m":{"ci":[17.423412908889823,19.65878482694102],"difference":18.542962413600087,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__slm-10m":{"ci":[14.224111495050982,16.29911483056891],"difference":15.244260934763345,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__supra-50m-base":{"ci":[7.358523605020158,9.038699445436707],"difference":8.211562757661675,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__veyra2-apricot-50m-base":{"ci":[8.539745385852653,10.31133646972243],"difference":9.431008597898938,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__atom-3-4m":{"ci":[12.414672706067238,14.51910829641859],"difference":13.523022309112628,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__falcon-h1-tiny-r-90m":{"ci":[6.088666706845199,8.086574541183792],"difference":7.079984111876598,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__gpt-s-1-4m":{"ci":[14.470574653613818,16.628850295397264],"difference":15.555665439254517,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__gpt-s2-5m":{"ci":[7.590743978294489,9.543732770147578],"difference":8.591214703538071,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__nanowhale-100m-base":{"ci":[9.06687024657007,10.940810819228764],"difference":10.002503456258717,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__pythia-160m":{"ci":[6.665984868286377,8.642212289460703],"difference":7.653797049885634,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__pythia-31m":{"ci":[9.944853555192541,11.916576975073232],"difference":10.919025791271682,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__slm-10m":{"ci":[6.635890360218745,8.602630427540772],"difference":7.620324312434939,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__supra-50m-base":{"ci":[-0.22231016068905624,1.4204925503246024],"difference":0.5876261353332725,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"gpt2-124m__veyra2-apricot-50m-base":{"ci":[0.9211716297796128,2.705342689004217],"difference":1.8070719755705327,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__atom-3-4m":{"ci":[27.58893756755036,30.042187485115143],"difference":28.789996315485123,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__baguettotron":{"ci":[12.443170128806557,14.63607099576427],"difference":13.526740651870819,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__falcon-h1-tiny-r-90m":{"ci":[21.180647286515022,23.545436928921546],"difference":22.34695811824909,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gemma-3-270m":{"ci":[3.2295084969038235,5.224620472855046],"difference":4.247314979070493,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt-s-1-4m":{"ci":[29.546430185030548,32.048984499243076],"difference":30.822639445627008,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt-s2-5m":{"ci":[22.692921220894892,25.060730091228283],"difference":23.858188709910568,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt-x2-125m":{"ci":[6.543548887485459,8.719161211825703],"difference":7.643037384044087,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt2-124m":{"ci":[14.143222563888733,16.4118593800478],"difference":15.266974006372491,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__mobilellm-r1-140m-base":{"ci":[10.873284350857205,13.02640742452202],"difference":11.955807929644676,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__nanowhale-100m-base":{"ci":[24.094314926003207,26.46428926562647],"difference":25.269477462631208,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__pythia-160m":{"ci":[21.76448313300386,24.12912979103644],"difference":22.920771056258122,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__pythia-31m":{"ci":[24.947883218281085,27.411184938190967],"difference":26.185999797644175,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__qwen2-5-0-5b":{"ci":[-0.7937443045238336,1.2109056554752804],"difference":0.19496537145225148,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"lfm2-350m__slm-10m":{"ci":[21.736389241473272,24.094793643932555],"difference":22.887298318807428,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__smollm2-135m":{"ci":[4.2639180400644365,6.304742227756106],"difference":5.305004387140445,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__supra-50m-base":{"ci":[14.717188310254356,16.936606168548764],"difference":15.854600141705763,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__veyra2-apricot-50m-base":{"ci":[15.922087105664474,18.21523002545234],"difference":17.074045981943023,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__atom-3-4m":{"ci":[15.724389468592625,17.9181576661227],"difference":16.83418838584044,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__baguettotron":{"ci":[0.6028456289645157,2.5682754519275246],"difference":1.5709327222261427,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__falcon-h1-tiny-r-90m":{"ci":[9.366761121353298,11.456567965416465],"difference":10.391150188604414,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__gpt-s-1-4m":{"ci":[17.72035213429454,19.952356120653356],"difference":18.86683151598233,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__gpt-s2-5m":{"ci":[10.837860366434668,12.916752286983545],"difference":11.902380780265888,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__gpt2-124m":{"ci":[2.3527040000432247,4.2604573197727165],"difference":3.3111660767278153,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__nanowhale-100m-base":{"ci":[12.244547029232189,14.383753052584806],"difference":13.313669532986532,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__pythia-160m":{"ci":[9.850460315653072,12.110903533239728],"difference":10.96496312661345,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__pythia-31m":{"ci":[13.173488573335227,15.260343800248487],"difference":14.230191867999498,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__slm-10m":{"ci":[9.834008908382621,11.962969758352465],"difference":10.931490389162754,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__supra-50m-base":{"ci":[2.9471642992203297,4.800180307345809],"difference":3.898792212061088,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__veyra2-apricot-50m-base":{"ci":[4.204117362859041,6.037751350789785],"difference":5.118238052298348,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"nanowhale-100m-base__atom-3-4m":{"ci":[2.5114800270993958,4.522962101886533],"difference":3.520518852853912,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"nanowhale-100m-base__gpt-s-1-4m":{"ci":[4.520390817955864,6.571451905864101],"difference":5.553161982995798,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"nanowhale-100m-base__pythia-31m":{"ci":[-0.04982526813426013,1.8445162857873265],"difference":0.9165223350129663,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"pythia-160m__atom-3-4m":{"ci":[4.737172915984907,7.018566565651529],"difference":5.8692252592269964,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-160m__gpt-s-1-4m":{"ci":[6.721804247693422,9.058153968300607],"difference":7.901868389368882,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-160m__gpt-s2-5m":{"ci":[-0.19073668035679597,2.031592137727357],"difference":0.9374176536524387,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"pythia-160m__nanowhale-100m-base":{"ci":[1.24315052655882,3.4148330794518693],"difference":2.348706406373083,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-160m__pythia-31m":{"ci":[2.148599543800454,4.357216741831833],"difference":3.2652287413860495,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-31m__atom-3-4m":{"ci":[1.6057485508016338,3.598754829970397],"difference":2.6039965178409457,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-31m__gpt-s-1-4m":{"ci":[3.553404497040384,5.681548302264185],"difference":4.636639647982832,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__atom-3-4m":{"ci":[27.429065503800224,29.758126578860814],"difference":28.59503094403287,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__baguettotron":{"ci":[12.233474837251677,14.410463289394922],"difference":13.331775280418565,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__falcon-h1-tiny-r-90m":{"ci":[21.042544193912452,23.292755794541097],"difference":22.151992746796836,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gemma-3-270m":{"ci":[3.1289293385506296,4.949670349757808],"difference":4.0523496076182415,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt-s-1-4m":{"ci":[29.358591326483403,31.787910953580926],"difference":30.627674074174756,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt-s2-5m":{"ci":[22.484367687263422,24.80643089248962],"difference":23.663223338458312,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt-x2-125m":{"ci":[6.504515401914428,8.401385534393647],"difference":7.448072012591836,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt2-124m":{"ci":[14.119663769992147,16.08811450435093],"difference":15.07200863492024,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__mobilellm-r1-140m-base":{"ci":[10.797843611952915,12.719193410831675],"difference":11.760842558192424,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__nanowhale-100m-base":{"ci":[23.99988085171323,26.190262920767886],"difference":25.074512091178956,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__pythia-160m":{"ci":[21.56004563074552,23.87905437950645],"difference":22.725805684805874,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__pythia-31m":{"ci":[24.841628881519235,27.14392774955085],"difference":25.991034426191924,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__slm-10m":{"ci":[21.573153354252433,23.791472841951883],"difference":22.69233294735518,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__smollm2-135m":{"ci":[4.182377294645956,6.041752403489344],"difference":5.110039015688194,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__supra-50m-base":{"ci":[14.674719371285,16.64642100282309],"difference":15.659634770253513,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__veyra2-apricot-50m-base":{"ci":[15.843063351497843,17.88305005638959],"difference":16.87908061049077,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__atom-3-4m":{"ci":[5.036334220813094,6.855128264261082],"difference":5.902697996677689,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__gpt-s-1-4m":{"ci":[6.926706396733146,8.94055735587147],"difference":7.935341126819576,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__gpt-s2-5m":{"ci":[0.06659882140217428,1.8222234812887246],"difference":0.970890391103133,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__nanowhale-100m-base":{"ci":[1.4914905145647108,3.2592393440485696],"difference":2.382179143823778,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__pythia-160m":{"ci":[-1.0366806397870105,1.1714324189900007],"difference":0.03347273745069435,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"slm-10m__pythia-31m":{"ci":[2.372327463312847,4.223426615282993],"difference":3.2987014788367444,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__atom-3-4m":{"ci":[22.34756564971431,24.615212198542363],"difference":23.484991928344677,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__baguettotron":{"ci":[7.21981046049257,9.250291115821636],"difference":8.221736264730373,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__falcon-h1-tiny-r-90m":{"ci":[15.998703266315285,18.097428607290865],"difference":17.04195373110864,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt-s-1-4m":{"ci":[24.306597878265215,26.660924664410892],"difference":25.517635058486558,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt-s2-5m":{"ci":[17.474818949264733,19.65927462835377],"difference":18.553184322770118,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt-x2-125m":{"ci":[1.520836731078651,3.1770188989552963],"difference":2.338032996903642,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt2-124m":{"ci":[9.01471391018264,10.912959371692486],"difference":9.961969619232047,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__mobilellm-r1-140m-base":{"ci":[5.754466156999673,7.575045374739086],"difference":6.65080354250423,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__nanowhale-100m-base":{"ci":[18.89929834073857,21.076546736195468],"difference":19.964473075490766,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__pythia-160m":{"ci":[16.533556862312427,18.762458693268172],"difference":17.61576666911768,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__pythia-31m":{"ci":[19.743832355893147,21.98849774424981],"difference":20.88099541050373,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__slm-10m":{"ci":[16.48898449494742,18.664225888028522],"difference":17.58229393166699,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__supra-50m-base":{"ci":[9.632456329100759,11.448445329599902],"difference":10.549595754565319,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__veyra2-apricot-50m-base":{"ci":[10.828312999027693,12.721441869861339],"difference":11.769041594802578,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__atom-3-4m":{"ci":[11.943975471387882,13.975141551589562],"difference":12.935396173779356,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__falcon-h1-tiny-r-90m":{"ci":[5.4933752538658185,7.494091690632844],"difference":6.4923579765433255,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__gpt-s-1-4m":{"ci":[13.913111223226107,16.026314781815554],"difference":14.968039303921243,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__gpt-s2-5m":{"ci":[7.055711440473382,8.973542914177806],"difference":8.0035885682048,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__nanowhale-100m-base":{"ci":[8.481265325554629,10.332046076912933],"difference":9.414877320925445,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__pythia-160m":{"ci":[5.988387846530205,8.160833376393306],"difference":7.0661709145523615,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__pythia-31m":{"ci":[9.341259784482242,11.317334294187644],"difference":10.33139965593841,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__slm-10m":{"ci":[6.057935337460307,8.006326905907825],"difference":7.032698177101667,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__veyra2-apricot-50m-base":{"ci":[0.352626714684538,2.1370426725718357],"difference":1.21944584023726,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__atom-3-4m":{"ci":[10.627668222625106,12.751269936412879],"difference":11.715950333542096,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__falcon-h1-tiny-r-90m":{"ci":[4.269879876321943,6.237640792156269],"difference":5.272912136306066,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__gpt-s-1-4m":{"ci":[12.499782605339313,14.838887566245907],"difference":13.748593463683981,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__gpt-s2-5m":{"ci":[5.8032698087522885,7.724554386452487],"difference":6.784142727967539,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__nanowhale-100m-base":{"ci":[7.226025832871106,9.190466152028234],"difference":8.195431480688185,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__pythia-160m":{"ci":[4.834200955167154,6.909557402281555],"difference":5.846725074315101,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__pythia-31m":{"ci":[8.142263967111687,10.097664917317006],"difference":9.11195381570115,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__slm-10m":{"ci":[4.863643976660482,6.735987982048942],"difference":5.813252336864407,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"}}'),Vo="2026-07-20T07:29:29.741057+00:00",Xo=JSON.parse('[{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.39590443686006827,"chance":0.25,"ci":[15.696814562002272,23.321956769055745],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":19.45392491467577},"arc_easy":{"accuracy":0.6641414141414141,"chance":0.25,"ci":[52.637485970819306,57.744107744107744],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":55.21885521885522},"blimp":{"accuracy":0.7906593253968254,"categories":{"Anaphor agreement":0.977,"Argument structure":0.7756666666666666,"Binding":0.7418571428571428,"Control / raising":0.8074,"Determiner-noun agreement":0.9377500000000001,"Ellipsis":0.8654999999999999,"Filler-gap":0.7779999999999999,"Irregular forms":0.728,"Island effects":0.739,"NPI licensing":0.5485714285714286,"Quantifiers":0.7414999999999999,"Subject-verb agreement":0.8476666666666666},"category_count":12,"chance":0.5,"ci":[57.22199222883596,58.42621676587302],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":58.13186507936508},"commonsense_qa":{"accuracy":0.45045045045045046,"chance":0.2,"ci":[27.927927927927925,34.88943488943489],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":31.306306306306304},"hellaswag":{"accuracy":0.4898426608245369,"chance":0.25,"ci":[30.61143198566022,33.25433180641306],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":31.97902144327159},"piqa":{"accuracy":0.6958650707290533,"chance":0.5,"ci":[35.03808487486397,43.2018498367791],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":39.17301414581067}},"complete":true,"composite":{"ci":[42.03623437890757,44.21094708361269],"classification":"reliably above chance","score":43.17876224644195},"id":"lfm2-350m","name":"LFM2 350M","params_m":354.484,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":354483968,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7e746d33d940>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[1,2],"release_date":"2025-07-10","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154647Z-v1.1","source":"LiquidAI/LFM2-350M","track":"instruction","updated_at":"2026-07-16T15:46:57.444261+00:00","vocab_size":65536},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.3199658703071672,"chance":0.25,"ci":[5.802047781569963,13.083048919226394],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":9.328782707622297},"arc_easy":{"accuracy":0.5845959595959596,"chance":0.25,"ci":[41.86307519640854,47.1955667789001],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":44.61279461279461},"blimp":{"accuracy":0.8435170304232805,"categories":{"Anaphor agreement":0.986,"Argument structure":0.8395555555555556,"Binding":0.7834285714285715,"Control / raising":0.8469999999999999,"Determiner-noun agreement":0.95575,"Ellipsis":0.8895,"Filler-gap":0.7902857142857144,"Irregular forms":0.9515,"Island effects":0.694125,"NPI licensing":0.7531428571428572,"Quantifiers":0.7372500000000001,"Subject-verb agreement":0.8946666666666667},"category_count":12,"chance":0.5,"ci":[67.90119460978836,68.90242162698412],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":68.70340608465611},"commonsense_qa":{"accuracy":0.4095004095004095,"chance":0.2,"ci":[22.91154791154791,29.67086404586403],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":26.187551187551186},"hellaswag":{"accuracy":0.5212109141605258,"chance":0.25,"ci":[34.846976033990565,37.50282148310429],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":36.161455221403436},"piqa":{"accuracy":0.6942328618063112,"chance":0.5,"ci":[34.71164309031556,42.763873775843294],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":38.84657236126223}},"complete":true,"composite":{"ci":[41.86348233604306,43.96668012300082],"classification":"reliably above chance","score":42.989729225325554},"id":"qwen2-5-0-5b","name":"Qwen2.5 0.5B","params_m":494,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":494032768,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x72645c9311c0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[1,2],"release_date":"2024-09-15","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154647Z-v1.1","source":"Qwen/Qwen2.5-0.5B","track":"base","updated_at":"2026-07-16T15:47:02.275688+00:00","vocab_size":151936},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.27986348122866894,"chance":0.25,"ci":[0.5688282138794095,7.281001137656425],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":3.9817974971558585},"arc_easy":{"accuracy":0.5728114478114478,"chance":0.25,"ci":[40.404040404040394,45.62289562289561],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":43.0415263748597},"blimp":{"accuracy":0.8213405423280423,"categories":{"Anaphor agreement":0.99,"Argument structure":0.7935555555555555,"Binding":0.7918571428571427,"Control / raising":0.8374,"Determiner-noun agreement":0.956,"Ellipsis":0.87,"Filler-gap":0.7927142857142858,"Irregular forms":0.9295,"Island effects":0.68875,"NPI licensing":0.668142857142857,"Quantifiers":0.7084999999999999,"Subject-verb agreement":0.8296666666666667},"category_count":12,"chance":0.5,"ci":[63.49301306216931,64.53917559523809],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":64.26810846560846},"commonsense_qa":{"accuracy":0.4258804258804259,"chance":0.2,"ci":[24.754299754299755,31.71580671580671],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":28.235053235053236},"hellaswag":{"accuracy":0.4144592710615415,"chance":0.25,"ci":[20.65325632344155,23.29582420500564],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":21.927902808205534},"piqa":{"accuracy":0.6849836779107725,"chance":0.5,"ci":[32.86180631120783,41.240478781284004],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":36.99673558215451}},"complete":true,"composite":{"ci":[37.774028206553304,39.9138118144519],"classification":"reliably above chance","score":38.912790926162764},"id":"gemma-3-270m","name":"Gemma 3 270M","params_m":270,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"32","batch_sizes":[],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":268098176,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7ff024f928e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[3,3],"release_date":"2025-08-05","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154647Z-v1.1","source":"google/gemma-3-270m","track":"base","updated_at":"2026-07-16T15:46:49.790812+00:00","vocab_size":262144},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.29692832764505117,"chance":0.25,"ci":[2.6166097838452806,9.670079635949946],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":6.2571103526734895},"arc_easy":{"accuracy":0.5871212121212122,"chance":0.25,"ci":[42.255892255892256,47.53086419753087],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":44.949494949494955},"blimp":{"accuracy":0.8136013227513228,"categories":{"Anaphor agreement":0.977,"Argument structure":0.8337777777777778,"Binding":0.7632857142857142,"Control / raising":0.8392000000000002,"Determiner-noun agreement":0.968,"Ellipsis":0.8745,"Filler-gap":0.7977142857142857,"Irregular forms":0.8380000000000001,"Island effects":0.6892499999999999,"NPI licensing":0.6485714285714286,"Quantifiers":0.67025,"Subject-verb agreement":0.8636666666666667},"category_count":12,"chance":0.5,"ci":[61.932142030423265,63.0667154431217],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":62.72026455026456},"commonsense_qa":{"accuracy":0.35544635544635544,"chance":0.2,"ci":[16.152231777231773,22.81173218673217],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":19.43079443079443},"hellaswag":{"accuracy":0.43218482374029077,"chance":0.25,"ci":[23.016331408086035,25.592511451902016],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":24.291309832038767},"piqa":{"accuracy":0.6817192600652884,"chance":0.5,"ci":[32.097388465723604,40.58759521218715],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":36.34385201305768}},"complete":true,"composite":{"ci":[36.68113506339092,38.849860312348774],"classification":"reliably above chance","score":37.843491883620345},"id":"smollm2-135m","name":"SmolLM2 135M","params_m":135,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":134515008,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7162a1fca3e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[4,4],"release_date":"2024-10-31","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154647Z-v1.1","source":"HuggingFaceTB/SmolLM2-135M","track":"base","updated_at":"2026-07-16T15:47:05.346313+00:00","vocab_size":49152},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.27559726962457337,"chance":0.25,"ci":[-0.11376564277588337,7.053469852104666],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":3.4129692832764498},"arc_easy":{"accuracy":0.5164141414141414,"chance":0.25,"ci":[32.82687991021324,38.21548821548822],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":35.52188552188553},"blimp":{"accuracy":0.8291201719576721,"categories":{"Anaphor agreement":0.9864999999999999,"Argument structure":0.826111111111111,"Binding":0.7557142857142857,"Control / raising":0.8231999999999999,"Determiner-noun agreement":0.9581249999999999,"Ellipsis":0.8534999999999999,"Filler-gap":0.7958571428571429,"Irregular forms":0.923,"Island effects":0.6868749999999999,"NPI licensing":0.7461428571428571,"Quantifiers":0.70725,"Subject-verb agreement":0.8871666666666665},"category_count":12,"chance":0.5,"ci":[64.96420122354498,66.0613258928571],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":65.82403439153441},"commonsense_qa":{"accuracy":0.3464373464373464,"chance":0.2,"ci":[14.926289926289922,21.580671580671574],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":18.3046683046683},"hellaswag":{"accuracy":0.4051981676956781,"chance":0.25,"ci":[19.431388169687313,21.941512314943907],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":20.693089026090416},"piqa":{"accuracy":0.6708378672470077,"chance":0.5,"ci":[29.706202393906423,38.41131664853101],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":34.16757344940153}},"complete":true,"composite":{"ci":[34.34048401085747,36.481411360993164],"classification":"reliably above chance","score":35.52389451013529},"id":"gpt-x2-125m","name":"GPT-X2 125M","params_m":125.081664,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":125081664,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x73557ef813a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[5,5],"release_date":"2026-03-25","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154647Z-v1.1","source":"AxiomicLabs/GPT-X2-125M","track":"base","updated_at":"2026-07-16T15:46:54.567291+00:00","vocab_size":32768},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/mobilellm-r1-140m-base/20260716T161918Z/models__mobilellm-r1-140m-base/results_2026-07-16T18-27-38.828116.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/mobilellm-r1-140m-base/20260716T161918Z"},"benchmarks":{"arc_challenge":{"accuracy":0.24146757679180889,"chance":0.25,"ci":[-4.550625711035269,2.2753128555176305],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-1.1376564277588153},"arc_easy":{"accuracy":0.49915824915824913,"chance":0.25,"ci":[30.47138047138047,35.746352413019075],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":33.22109988776655},"blimp":{"accuracy":0.8078363425925926,"categories":{"Anaphor agreement":0.9735,"Argument structure":0.7977777777777777,"Binding":0.7627142857142858,"Control / raising":0.8108000000000001,"Determiner-noun agreement":0.9315,"Ellipsis":0.892,"Filler-gap":0.767,"Irregular forms":0.9495,"Island effects":0.6956249999999999,"NPI licensing":0.6222857142857142,"Quantifiers":0.6245,"Subject-verb agreement":0.8668333333333335},"category_count":12,"chance":0.5,"ci":[60.729346891534426,61.79221445105819],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":61.567268518518524},"commonsense_qa":{"accuracy":0.3546273546273546,"chance":0.2,"ci":[15.950040950040947,22.809172809172804],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":19.328419328419326},"hellaswag":{"accuracy":0.3406691894045011,"chance":0.25,"ci":[10.893580296089754,13.297815840138082],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":12.089225253933478},"piqa":{"accuracy":0.6327529923830251,"chance":0.5,"ci":[22.19804134929271,30.794341675734493],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":26.550598476605014}},"complete":true,"composite":{"ci":[30.06941936355877,32.21641211384041],"classification":"reliably above chance","score":31.211043675010572},"id":"mobilellm-r1-140m-base","name":"MobileLLM-R1 140M Base","params_m":140,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":140248512,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7166fab5e020>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[6,6],"release_date":"2025-09-10","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/mobilellm-r1-140m-base/20260716T161918Z","source":"facebook/MobileLLM-R1-140M-base","track":"base","updated_at":"2026-07-16T16:27:44.727364+00:00","vocab_size":128256},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/models__baguettotron/results_2026-07-20T01-21-17.781285.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z"},"benchmarks":{"arc_challenge":{"accuracy":0.3037542662116041,"chance":0.25,"ci":[3.7542662116040924,10.58304891922638],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":7.16723549488055},"arc_easy":{"accuracy":0.5058922558922558,"chance":0.25,"ci":[31.48148148148148,36.700336700336706],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":34.11896745230078},"blimp":{"accuracy":0.7847246362433862,"categories":{"Anaphor agreement":0.8355,"Argument structure":0.7777777777777778,"Binding":0.7462857142857143,"Control / raising":0.7584,"Determiner-noun agreement":0.941,"Ellipsis":0.88,"Filler-gap":0.7588571428571429,"Irregular forms":0.915,"Island effects":0.630375,"NPI licensing":0.623,"Quantifiers":0.7064999999999999,"Subject-verb agreement":0.8439999999999999},"category_count":12,"chance":0.5,"ci":[56.01578158068786,57.203216600529124],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":56.94492724867723},"commonsense_qa":{"accuracy":0.3063063063063063,"chance":0.2,"ci":[10.012285012285009,16.56429156429156],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":13.288288288288284},"hellaswag":{"accuracy":0.35391356303525195,"chance":0.25,"ci":[12.607050388368853,15.050122817499838],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":13.855141738033595},"piqa":{"accuracy":0.6207834602829162,"chance":0.5,"ci":[19.583786724700754,28.726877040261158],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":24.15669205658324}},"complete":true,"composite":{"ci":[28.428847552063168,30.68468269023897],"classification":"reliably above chance","score":29.63151077505659},"id":"baguettotron","name":"Baguettotron","params_m":321,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":"dc3c1cf","inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":320956992,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7ed43ed7b560>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[7,7],"release_date":"2025-11-10","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z","source":"PleIAs/Baguettotron","track":"instruction","updated_at":"2026-07-19T23:21:23.658293+00:00","vocab_size":65536},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.22440273037542663,"chance":0.25,"ci":[-6.598407281001136,-0.22753128555176305],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.4129692832764498},"arc_easy":{"accuracy":0.3947811447811448,"chance":0.25,"ci":[16.665263748597084,21.885521885521886],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":19.304152637485974},"blimp":{"accuracy":0.8377077380952381,"categories":{"Anaphor agreement":0.997,"Argument structure":0.8356666666666667,"Binding":0.79,"Control / raising":0.8006,"Determiner-noun agreement":0.953875,"Ellipsis":0.848,"Filler-gap":0.8117142857142856,"Irregular forms":0.958,"Island effects":0.693625,"NPI licensing":0.7674285714285715,"Quantifiers":0.7177500000000001,"Subject-verb agreement":0.8788333333333332},"category_count":12,"chance":0.5,"ci":[66.76148528439154,67.78422933201058],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":67.54154761904762},"commonsense_qa":{"accuracy":0.29811629811629814,"chance":0.2,"ci":[9.193284193284189,15.54054054054054],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":12.264537264537264},"hellaswag":{"accuracy":0.31139215295757816,"chance":0.25,"ci":[7.003916882427141,9.380601473809998],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":8.185620394343756},"piqa":{"accuracy":0.6251360174102285,"chance":0.5,"ci":[20.67464635473342,29.37976060935801],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":25.0272034820457}},"complete":true,"composite":{"ci":[26.76217046436712,28.856641117490366],"classification":"reliably above chance","score":27.871165134240197},"id":"gpt2-124m","name":"GPT-2 124M","params_m":124,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false},"model_dtype":"torch.float32","model_num_parameters":124439808,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x707d2cd280e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[8,9],"release_date":"2022-03-02","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154647Z-v1.1","source":"openai-community/gpt2","track":"base","updated_at":"2026-07-16T15:46:55.978943+00:00","vocab_size":50257},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.24914675767918087,"chance":0.25,"ci":[-3.526734926052333,3.1854379977246903],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-0.11376564277588337},"arc_easy":{"accuracy":0.4562289562289562,"chance":0.25,"ci":[24.74747474747475,30.1921997755331],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":27.497194163860826},"blimp":{"accuracy":0.7947869047619047,"categories":{"Anaphor agreement":0.913,"Argument structure":0.7863333333333333,"Binding":0.7452857142857142,"Control / raising":0.8018000000000001,"Determiner-noun agreement":0.9412499999999999,"Ellipsis":0.857,"Filler-gap":0.7838571428571429,"Irregular forms":0.8845000000000001,"Island effects":0.63425,"NPI licensing":0.624,"Quantifiers":0.6865,"Subject-verb agreement":0.8796666666666667},"category_count":12,"chance":0.5,"ci":[57.94834143518519,59.13693105158729],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":58.957380952380944},"commonsense_qa":{"accuracy":0.29975429975429974,"chance":0.2,"ci":[9.193284193284189,15.74529074529074],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":12.469287469287464},"hellaswag":{"accuracy":0.31627165903206533,"chance":0.25,"ci":[7.6279625572595116,9.991369581092746],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":8.836221204275377},"piqa":{"accuracy":0.6169749727965179,"chance":0.5,"ci":[18.933623503808494,27.967899891186054],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":23.394994559303584}},"complete":true,"composite":{"ci":[26.15776710993005,28.30631083392145],"classification":"reliably above chance","score":27.338042663290054},"id":"supra-50m-base","name":"Supra 50M Base","params_m":51.78624,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":51786240,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T153917Z/tokenizer_compat","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T153917Z/tokenizer_compat","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7e92921a09a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[8,9],"release_date":"2026-05-21","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154647Z-v1.1","source":"SupraLabs/Supra-50M-Base","track":"base","updated_at":"2026-07-16T15:47:07.077007+00:00","vocab_size":32000},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/models__veyra2-apricot-50m-base/results_2026-07-16T18-37-33.567087.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z"},"benchmarks":{"arc_challenge":{"accuracy":0.23464163822525597,"chance":0.25,"ci":[-5.233219567690558,1.137656427758819],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-2.047781569965871},"arc_easy":{"accuracy":0.4276094276094276,"chance":0.25,"ci":[20.93153759820427,26.430976430976433],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":23.681257014590347},"blimp":{"accuracy":0.7935405423280423,"categories":{"Anaphor agreement":0.9115,"Argument structure":0.7822222222222223,"Binding":0.7711428571428571,"Control / raising":0.7988,"Determiner-noun agreement":0.933125,"Ellipsis":0.849,"Filler-gap":0.7371428571428572,"Irregular forms":0.907,"Island effects":0.676375,"NPI licensing":0.6524285714285715,"Quantifiers":0.67225,"Subject-verb agreement":0.8315},"category_count":12,"chance":0.5,"ci":[57.73771858465608,58.93828703703703],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":58.708108465608454},"commonsense_qa":{"accuracy":0.28665028665028663,"chance":0.2,"ci":[7.657657657657656,14.004914004914006],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":10.831285831285827},"hellaswag":{"accuracy":0.3127862975502888,"chance":0.25,"ci":[7.203080395671517,9.540264223594237],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":8.371506340038508},"piqa":{"accuracy":0.6169749727965179,"chance":0.5,"ci":[19.0424374319913,27.856365614798694],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":23.394994559303584}},"complete":true,"composite":{"ci":[24.919591817425154,27.053619924780357],"classification":"reliably above chance","score":26.088907580286257},"id":"veyra2-apricot-50m-base","name":"Veyra2 Apricot 50M Base","params_m":49.3,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"64","batch_sizes":[],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":49303040,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7c9c447b31a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[10,10],"release_date":"2026-06-14","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z","source":"veyra-ai/Veyra2-Apricot-50M-Base","track":"base","updated_at":"2026-07-16T16:37:38.898399+00:00","vocab_size":8192},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/falcon-h1-tiny-r-90m/20260720T072155Z/models__falcon-h1-tiny-r-90m/results_2026-07-20T09-28-46.371565.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/falcon-h1-tiny-r-90m/20260720T072155Z"},"benchmarks":{"arc_challenge":{"accuracy":0.24914675767918087,"chance":0.25,"ci":[-3.415813424345847,3.1854379977246903],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-0.11376564277588337},"arc_easy":{"accuracy":0.3926767676767677,"chance":0.25,"ci":[16.273849607182942,21.49410774410773],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":19.023569023569024},"blimp":{"accuracy":0.7126083333333333,"categories":{"Anaphor agreement":0.8009999999999999,"Argument structure":0.6073333333333334,"Binding":0.7232857142857142,"Control / raising":0.6587999999999999,"Determiner-noun agreement":0.8799999999999999,"Ellipsis":0.871,"Filler-gap":0.7327142857142858,"Irregular forms":0.781,"Island effects":0.5905,"NPI licensing":0.704,"Quantifiers":0.44050000000000006,"Subject-verb agreement":0.7611666666666667},"category_count":12,"chance":0.5,"ci":[41.63552711640213,42.98391468253967],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":42.52166666666666},"commonsense_qa":{"accuracy":0.2833742833742834,"chance":0.2,"ci":[7.350532350532346,13.595413595413595],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":10.42178542178542},"hellaswag":{"accuracy":0.30770762796255724,"chance":0.25,"ci":[6.552479585739893,8.876053906924254],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":7.694350395007632},"piqa":{"accuracy":0.6077257889009793,"chance":0.5,"ci":[16.97497279651796,25.897714907508163],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":21.545157780195858}},"complete":true,"composite":{"ci":[19.64362638928377,21.818589193772866],"classification":"reliably above chance","score":20.800441105410517},"id":"falcon-h1-tiny-r-90m","name":"Falcon H1 Tiny R 90M","params_m":90,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":"dc3c1cf","inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":91131072,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7cc583ab0cc0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[11,13],"release_date":"2026-01-12","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/falcon-h1-tiny-r-90m/20260720T072155Z","source":"tiiuae/Falcon-H1-Tiny-R-90M","track":"instruction","updated_at":"2026-07-20T07:28:52.389064+00:00","vocab_size":32768},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.2354948805460751,"chance":0.25,"ci":[-5.1222980659840704,1.251422070534695],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-1.9340159271899877},"arc_easy":{"accuracy":0.3569023569023569,"chance":0.25,"ci":[11.560044893378228,16.835016835016834],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":14.253647586980918},"blimp":{"accuracy":0.772000496031746,"categories":{"Anaphor agreement":0.8959999999999999,"Argument structure":0.7603333333333333,"Binding":0.7562857142857142,"Control / raising":0.757,"Determiner-noun agreement":0.89825,"Ellipsis":0.7855,"Filler-gap":0.7295714285714286,"Irregular forms":0.9505,"Island effects":0.508875,"NPI licensing":0.7078571428571429,"Quantifiers":0.6639999999999999,"Subject-verb agreement":0.8498333333333333},"category_count":12,"chance":0.5,"ci":[53.51157457010582,54.73998561507939],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":54.4000992063492},"commonsense_qa":{"accuracy":0.2596232596232596,"chance":0.2,"ci":[4.484029484029484,10.62653562653563],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":7.452907452907449},"hellaswag":{"accuracy":0.2731527584146584,"chance":0.25,"ci":[1.958109274380932,4.228905264555535],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":3.087034455287789},"piqa":{"accuracy":0.5707290533188248,"chance":0.5,"ci":[9.681719260065288,18.50108813928181],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":14.14581066376497}},"complete":true,"composite":{"ci":[19.071533266309793,21.225442487059897],"classification":"reliably above chance","score":20.26599982402355},"id":"slm-10m","name":"SLM 10M","params_m":9.96864,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":9968640,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T153812Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T153812Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7e9023f9ccc0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[11,13],"release_date":"2026-06-13","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154647Z-v1.1","source":"liodon-ai/slm-10m","track":"base","updated_at":"2026-07-16T15:47:03.814706+00:00","vocab_size":8192},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.23037542662116042,"chance":0.25,"ci":[-5.688282138794084,0.5688282138794095],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-2.616609783845277},"arc_easy":{"accuracy":0.3686868686868687,"chance":0.25,"ci":[13.18742985409652,18.350168350168353],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":15.824915824915822},"blimp":{"accuracy":0.7286577050264551,"categories":{"Anaphor agreement":0.9245000000000001,"Argument structure":0.7542222222222222,"Binding":0.7448571428571429,"Control / raising":0.7462,"Determiner-noun agreement":0.826125,"Ellipsis":0.759,"Filler-gap":0.6882857142857144,"Irregular forms":0.723,"Island effects":0.631,"NPI licensing":0.5852857142857142,"Quantifiers":0.63075,"Subject-verb agreement":0.7306666666666667},"category_count":12,"chance":0.5,"ci":[44.78496957671956,46.238928736772515],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":45.73154100529102},"commonsense_qa":{"accuracy":0.2932022932022932,"chance":0.2,"ci":[8.37428337428337,14.823914823914825],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":11.650286650286647},"hellaswag":{"accuracy":0.30501892053375823,"chance":0.25,"ci":[6.167430126800774,8.504282015534756],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":7.335856071167764},"piqa":{"accuracy":0.5843307943416758,"chance":0.5,"ci":[12.51360174102285,21.218715995647443],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":16.866158868335155}},"complete":true,"composite":{"ci":[19.111437730092515,21.23647682872686],"classification":"reliably above chance","score":20.2119024553024},"id":"pythia-160m","name":"Pythia 160M","params_m":160,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":162322944,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x71da3d32f9c0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[11,14],"release_date":"2023-02-08","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154647Z-v1.1","source":"EleutherAI/pythia-160m","track":"base","updated_at":"2026-07-16T15:46:58.993182+00:00","vocab_size":50304},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.22184300341296928,"chance":0.25,"ci":[-6.939704209328782,-0.6825938566552892],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.754266211604096},"arc_easy":{"accuracy":0.335016835016835,"chance":0.25,"ci":[8.585858585858587,13.916947250280586],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":11.335578002244665},"blimp":{"accuracy":0.7732386243386244,"categories":{"Anaphor agreement":0.9405,"Argument structure":0.7917777777777779,"Binding":0.732857142857143,"Control / raising":0.7488,"Determiner-noun agreement":0.902375,"Ellipsis":0.7815000000000001,"Filler-gap":0.7285714285714285,"Irregular forms":0.915,"Island effects":0.538875,"NPI licensing":0.7108571428571429,"Quantifiers":0.63225,"Subject-verb agreement":0.8555000000000001},"category_count":12,"chance":0.5,"ci":[53.86334275793652,55.039749007936514],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":54.64772486772489},"commonsense_qa":{"accuracy":0.24815724815724816,"chance":0.2,"ci":[3.1505937755937747,9.193284193284189],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":6.019656019656018},"hellaswag":{"accuracy":0.27544313881696875,"chance":0.25,"ci":[2.2638252672110504,4.5083980614751376],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":3.392418508929166},"piqa":{"accuracy":0.5647442872687704,"chance":0.5,"ci":[8.26985854189337,17.41022850924918],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":12.94885745375407}},"complete":true,"composite":{"ci":[18.15426094733613,20.273369388072773],"classification":"reliably above chance","score":19.279532451183385},"id":"gpt-s2-5m","name":"GPT-S2 5M","params_m":5.384,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":5384258,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7891fe910ae0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[13,14],"release_date":"2026-06-19","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154647Z-v1.1","source":"AxiomicLabs/GPT-S2-5M","track":"base","updated_at":"2026-07-16T15:46:53.057636+00:00","vocab_size":4096},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/models__nanowhale-100m-base/results_2026-07-20T01-41-02.427304.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z"},"benchmarks":{"arc_challenge":{"accuracy":0.22440273037542663,"chance":0.25,"ci":[-6.484641638225256,-0.22753128555176305],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.4129692832764498},"arc_easy":{"accuracy":0.3728956228956229,"chance":0.25,"ci":[13.692480359147021,19.023569023569024],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":16.38608305274972},"blimp":{"accuracy":0.7091933531746032,"categories":{"Anaphor agreement":0.794,"Argument structure":0.7473333333333333,"Binding":0.7195714285714286,"Control / raising":0.6965999999999999,"Determiner-noun agreement":0.8567499999999999,"Ellipsis":0.7050000000000001,"Filler-gap":0.6524285714285715,"Irregular forms":0.917,"Island effects":0.495625,"NPI licensing":0.5134285714285715,"Quantifiers":0.64475,"Subject-verb agreement":0.7678333333333334},"category_count":12,"chance":0.5,"ci":[40.99927827380954,42.33725859788362],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":41.83867063492064},"commonsense_qa":{"accuracy":0.27764127764127766,"chance":0.2,"ci":[6.63390663390663,12.878787878787877],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":9.705159705159705},"hellaswag":{"accuracy":0.27126070503883687,"chance":0.25,"ci":[1.63977959237868,3.976631481112662],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":2.8347606718449163},"piqa":{"accuracy":0.573449401523395,"chance":0.5,"ci":[10.228509249183904,19.26006528835691],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":14.689880304678994}},"complete":true,"composite":{"ci":[16.768369462355462,18.88831197015816],"classification":"reliably above chance","score":17.857047764529888},"id":"nanowhale-100m-base","name":"nanowhale 100M Base","params_m":110,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":"dc3c1cf","inference_config":{"batch_size":"32","batch_sizes":[],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.float32","model_num_parameters":110369621,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x75209af931a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[15,16],"release_date":"2026-04-24","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z","source":"HuggingFaceTB/nanowhale-100m-base","track":"base","updated_at":"2026-07-19T23:41:08.863409+00:00","vocab_size":129280},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.2167235494880546,"chance":0.25,"ci":[-7.622298065984071,-1.1376564277588153],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-4.436860068259385},"arc_easy":{"accuracy":0.3409090909090909,"chance":0.25,"ci":[9.652076318742987,14.534231200897866],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":12.121212121212118},"blimp":{"accuracy":0.7167433531746031,"categories":{"Anaphor agreement":0.9285,"Argument structure":0.7563333333333333,"Binding":0.7057142857142856,"Control / raising":0.7332,"Determiner-noun agreement":0.857375,"Ellipsis":0.6745,"Filler-gap":0.6648571428571428,"Irregular forms":0.8815,"Island effects":0.535,"NPI licensing":0.45985714285714285,"Quantifiers":0.62375,"Subject-verb agreement":0.7803333333333334},"category_count":12,"chance":0.5,"ci":[42.51403703703704,43.85099421296294],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":43.34867063492063},"commonsense_qa":{"accuracy":0.26453726453726456,"chance":0.2,"ci":[5.098280098280095,11.138411138411136],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":8.067158067158068},"hellaswag":{"accuracy":0.27185819557857,"chance":0.25,"ci":[1.7725552678749275,4.069906393148773],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":2.9144260771426658},"piqa":{"accuracy":0.5680087051142546,"chance":0.5,"ci":[9.140369967355833,18.063112078346034],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":13.601741022850922}},"complete":true,"composite":{"ci":[15.828302485043055,17.965860323148288],"classification":"reliably above chance","score":16.92981203838316},"id":"pythia-31m","name":"Pythia 31M","params_m":31,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":30494720,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7ce4241c0180>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[15,16],"release_date":"2026-02-24","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154647Z-v1.1","source":"EleutherAI/pythia-31m","track":"base","updated_at":"2026-07-16T15:47:00.508307+00:00","vocab_size":50304},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.21843003412969283,"chance":0.25,"ci":[-7.394766780432309,-1.1376564277588153],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-4.209328782707622},"arc_easy":{"accuracy":0.33080808080808083,"chance":0.25,"ci":[8.361391694725029,13.24354657687991],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":10.774410774410777},"blimp":{"accuracy":0.6819768518518519,"categories":{"Anaphor agreement":0.6659999999999999,"Argument structure":0.7285555555555556,"Binding":0.713857142857143,"Control / raising":0.6869999999999999,"Determiner-noun agreement":0.842125,"Ellipsis":0.708,"Filler-gap":0.6594285714285714,"Irregular forms":0.845,"Island effects":0.5431250000000001,"NPI licensing":0.5187142857142858,"Quantifiers":0.53975,"Subject-verb agreement":0.7321666666666666},"category_count":12,"chance":0.5,"ci":[35.579373181216944,36.99308052248677],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":36.39537037037037},"commonsense_qa":{"accuracy":0.24733824733824733,"chance":0.2,"ci":[2.9484029484029453,9.193284193284189],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":5.917280917280914},"hellaswag":{"accuracy":0.27614021111332404,"chance":0.25,"ci":[2.356768240058423,4.547566885746535],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":3.4853614817765384},"piqa":{"accuracy":0.5571273122959739,"chance":0.5,"ci":[6.855277475516863,15.88683351468989],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":11.42546245919478}},"complete":true,"composite":{"ci":[13.263758832507774,15.316045445100848],"classification":"reliably above chance","score":14.327312373584093},"id":"atom-3-4m","name":"Atom 3.4M","params_m":3.4128,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true},"model_dtype":"torch.float32","model_num_parameters":3412800,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x77ee1d9145e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[17,17],"release_date":"2026-06-19","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154647Z-v1.1","source":"UniversalComputingResearch/Atom3.4m","track":"base","updated_at":"2026-07-16T15:46:48.573093+00:00","vocab_size":4096},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.22013651877133106,"chance":0.25,"ci":[-7.053469852104666,-0.6825938566552892],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.9817974971558585},"arc_easy":{"accuracy":0.31776094276094274,"chance":0.25,"ci":[6.509539842873177,11.503928170594838],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":9.034792368125698},"blimp":{"accuracy":0.6599056216931216,"categories":{"Anaphor agreement":0.6275,"Argument structure":0.6435555555555555,"Binding":0.7425714285714285,"Control / raising":0.6728000000000001,"Determiner-noun agreement":0.779625,"Ellipsis":0.6525000000000001,"Filler-gap":0.6531428571428571,"Irregular forms":0.8235,"Island effects":0.5603750000000001,"NPI licensing":0.41571428571428576,"Quantifiers":0.6217499999999999,"Subject-verb agreement":0.7258333333333332},"category_count":12,"chance":0.5,"ci":[31.20117526455027,32.64844031084656],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":31.981124338624323},"commonsense_qa":{"accuracy":0.23423423423423423,"chance":0.2,"ci":[1.5125921375921367,7.35309172809171],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":4.279279279279277},"hellaswag":{"accuracy":0.26847241585341564,"chance":0.25,"ci":[1.3075084644493165,3.5919139613622804],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":2.4629887804554182},"piqa":{"accuracy":0.5516866158868335,"chance":0.5,"ci":[5.658324265505987,14.692600652883549],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":10.33732317736671}},"complete":true,"composite":{"ci":[11.210655224075424,13.297313646044007],"classification":"reliably above chance","score":12.287040849624308},"id":"gpt-s-1-4m","name":"GPT-S 1.4M","params_m":1.426,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":1426176,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7c83fdb90c20>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[18,18],"release_date":"2026-06-01","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154647Z-v1.1","source":"AxiomicLabs/GPT-S-1.4M","track":"base","updated_at":"2026-07-16T15:46:51.370873+00:00","vocab_size":4096}]'),Wo="slm-benchmark-v1.1",en={comparisons:Go,generated_at:Vo,models:Xo,suite:Wo},tn="slm-benchmark-v1.1",_n="Illustrative data — not an evaluation",on=[{id:"demo-110m",name:"Illustrative Base 110M",params_m:110,track:"base",complete:!0,rank_range:[1,2],composite:{score:16.8,ci:[15.1,18.6],classification:"reliably above chance"},benchmarks:{hellaswag:{score:12.4,ci:[10.7,14.2]},piqa:{score:20.6,ci:[17.1,24]},arc_easy:{score:15,ci:[11.5,18.5]},arc_challenge:{score:7.1,ci:[3.4,10.7]},commonsense_qa:{score:8.9,ci:[5.2,12.6]},blimp:{score:27.2,ci:[24.8,29.6]}}},{id:"demo-160m",name:"Illustrative Base 160M",params_m:160,track:"base",complete:!0,rank_range:[1,2],composite:{score:17.7,ci:[16,19.5],classification:"reliably above chance"},benchmarks:{hellaswag:{score:13.9,ci:[12.1,15.6]},piqa:{score:22.1,ci:[18.7,25.4]},arc_easy:{score:16.3,ci:[12.9,19.7]},arc_challenge:{score:9.8,ci:[6.1,13.5]},commonsense_qa:{score:10.6,ci:[7.1,14]},blimp:{score:25.8,ci:[23.5,28.1]}}},{id:"demo-60m",name:"Illustrative Base 60M",params_m:60,track:"base",complete:!0,rank_range:[3,3],composite:{score:5.6,ci:[3.9,7.4],classification:"reliably above chance"},benchmarks:{hellaswag:{score:2.1,ci:[.5,3.8]},piqa:{score:7.3,ci:[4,10.5]},arc_easy:{score:3.8,ci:[.8,6.7]},arc_challenge:{score:1.4,ci:[-2.4,5.2]},commonsense_qa:{score:1.2,ci:[-2.1,4.4]},blimp:{score:13.6,ci:[11.2,16]}}}],nn={suite:tn,generated_at:_n,models:on};var an=te('<meta name="description" content="A carefully scored compact language model benchmark from Universal Computing Research."/>'),bt=te("<button> </button>"),sn=te('<th><button class="sort-button" type="button"> </button></th>'),ln=te('<tr class="empty-row"><td colspan="13">No complete results for this track yet.</td></tr>'),rn=te('<a class="model-link" target="_blank" rel="noreferrer"><span class="model-name"> </span></a>'),cn=te('<span class="model-name"> </span>'),mn=te('<span class="model-source"> </span>'),dn=te(" <span> </span>",1),pn=te("<td><!></td>"),un=te('<tr><td class="rank"> </td><td><!> <!></td><td class="metadata"> </td><td> </td><td class="metadata"> </td><td class="track-cell"><span> </span></td><!></tr>'),o_=te('<button type="button"><span class="bar-toggle-chevron" aria-hidden="true"></span></button>'),gn=tt('<line class="grid-line"></line><text class="axis-text" text-anchor="end"> </text>',1),hn=tt('<text class="axis-text" text-anchor="middle"> </text>'),bn=tt('<line class="frontier-line"></line>'),fn=tt('<circle r="7" tabindex="0" role="button"></circle>'),yn=te('<div class="chart-tooltip"><strong> </strong> <span> </span> <b> <em> </em></b></div>'),vn=te('<div class="scatter-area" role="group"><svg preserveAspectRatio="xMidYMid meet" role="img"><text class="axis-title" x="7" text-anchor="middle">Chance-normalized score</text><!><line class="axis-line"></line><!><text class="axis-title" dy="16" text-anchor="middle">Parameters (square-root scale)</text><!><!></svg> <!></div>'),wn=te('<div class="empty-chart">Completed evaluations will appear here.</div>'),xn=te("<i></i>"),kn=te('<div class="bar-row"><span class="bar-rank"> </span> <span class="bar-name"> </span> <div class="bar-track"><i class="zero-line"></i><i></i><!></div> <strong> </strong></div>'),Pn=te('<div class="bar-list"></div> <!>',1),jn=te('<div class="empty-chart short">No measured values for this selection.</div>'),qn=te('<main><section class="hero" id="top"><div class="hero-copy"><div class="hero-kicker"><p class="eyebrow">Small language model evaluation</p> <span class="edition">Research index</span></div> <h1>Compact<br/>Language Model<br/><span>Index</span></h1> <p class="hero-description">A fixed evaluation of compact language models across six benchmarks, with chance-normalized scores and 95% intervals</p> <a class="hero-link" href="#leaderboard-title">Explore the leaderboard</a></div></section> <section class="summary-grid" aria-label="Suite summary"><article><span class="summary-index">Evaluation breadth</span><div class="summary-value"><strong>6</strong><span>core benchmarks</span></div></article> <article><span class="summary-index">Linguistic depth</span><div class="summary-value"><strong>12</strong><span>BLiMP categories</span></div></article> <article><span class="summary-index">Current sample</span><div class="summary-value"><strong> </strong><span>models measured</span></div></article></section> <section class="leaderboard-section" aria-labelledby="leaderboard-title"><div class="section-heading table-heading"><div><p class="eyebrow">Ranked results</p> <h2 id="leaderboard-title">Core leaderboard</h2></div> <div class="leaderboard-controls"><div class="filter-stack"><div class="top-filter-row"><div class="filters view-filter"><button type="button">Simplified view</button></div> <div class="filters" aria-label="Model track filter"></div></div> <div class="size-filters" aria-label="Maximum parameter count"></div></div></div></div> <div class="table-wrap"><table><thead><tr><th>Rank</th><th><button class="sort-button" type="button">Model</button></th><th class="metadata-head" title="Hugging Face repository creation date">Release date</th><th><button class="sort-button" type="button">Parameters</button></th><th class="metadata-head">Vocab. size</th><th class="track-head">Track</th><th><button class="sort-button" type="button">Average score <span>95% CI</span></button></th><!></tr></thead><tbody><!></tbody></table></div> <!></section> <section class="chart-controls" aria-label="Chart metric selector"><div><p class="eyebrow">Chart metric</p> <p class="section-note">Select a benchmark to update both the scaling chart and score comparison below.</p></div> <div class="task-tabs" aria-label="Benchmark score selector"></div></section> <section class="panel landscape" aria-labelledby="landscape-title"><div class="section-heading"><div><p class="eyebrow">Model scaling</p> <h2 id="landscape-title"> </h2></div> <div class="legend"><span class="legend-dot base"></span>Base <span class="legend-dot instruction"></span>Instruction-tuned <span class="frontier-label"><span class="frontier-key"></span>Pareto trend</span></div></div> <!></section> <section class="panel score-panel" aria-labelledby="scores-title"><div class="section-heading compact-heading"><div><h2 id="scores-title">Model score comparison</h2> <p class="section-note">The top 10 matching models are shown by default. This comparison uses the shared metric selected above.</p></div></div> <!></section> <section class="bottom-grid"><article class="panel method-card"><p class="eyebrow">Scoring method</p> <h3>Chance-normalized composite score</h3> <p class="formula">100 × <span>(accuracy − chance)</span> / <span>(1 − chance)</span></p> <p>For each benchmark, the observed result is normalized against its chance level: chance maps to 0 and a perfect result maps to 100.</p> <p>HellaSwag, PIQA, and CommonsenseQA form a commonsense domain worth 50% of the composite, with each benchmark weighted equally within that domain. Science and world knowledge contribute 25% after combining ARC-Easy (75%) and ARC-Challenge (25%). BLiMP contributes the remaining 25% for grammatical competence and is macro-averaged across its 12 linguistic categories.</p> <p>CommonsenseQA scores the likelihood of each full answer text with length normalization, rather than scoring the A–E answer label. This prevents a model’s letter preference from being mistaken for commonsense capability.</p> <p>Each benchmark interval is a 95% percentile bootstrap. Model comparisons use paired bootstrap resampling on matched evaluation items. Results below chance remain negative, and rank ranges reflect comparisons that do not distinguish neighbouring models.</p></article> <article class="panel protocol-card"><p class="eyebrow">Evaluation protocol</p> <h2>Curated evaluation</h2> <p>CompactLMIndex is a curated benchmark, not an open intake list. We are establishing a Pareto-frontier baseline across model sizes and highlighting training or inference strategies that put a model ahead of its peers at a comparable parameter count.</p> <ul><li>Complete core suite required for a rank</li><li>Zero-shot, non-chat log-likelihood prompts</li><li>Paired bootstrap comparisons on matched questions</li></ul> <p>Every published result is independently checked through our internal verification process. Final inclusion and presentation decisions remain with Universal Computing Research.</p> <p>If you have interesting results for a model you think is suitable for this benchmark, you can open a discussion on the Space for consideration.</p></article></section> <section class="panel additional-card" aria-labelledby="credits-title"><div><p class="eyebrow">Credits &amp; scope</p> <h2 id="credits-title">Efficiency at smaller scales</h2></div> <div class="credits-content"><p>CompactLMIndex looks at smaller language models. At this scale, architecture, training, and inference choices can make a real difference. Every model follows the same evaluation protocol, so results are compared fairly rather than by size alone.</p> <p class="credits-attribution">Inspired by the <a href="https://huggingface.co/open-llm-leaderboard" target="_blank" rel="noreferrer">Open LLM Leaderboard</a>, <a href="https://huggingface.co/spaces/AxiomicLabs/Open_SLM_Leaderboard" target="_blank" rel="noreferrer">Axiomic Labs Open SLM Leaderboard</a>, and <a href="https://github.com/openai/parameter-golf" target="_blank" rel="noreferrer">OpenAI Parameter Golf</a>.</p></div></section> <footer><p>Compact Language Model Index by <a href="https://huggingface.co/UniversalComputingResearch" target="_blank" rel="noreferrer">Universal Computing Research</a></p></footer></main>');function On(t,_){Po(_,!1);const s=[{id:"average",label:"Average"},{id:"hellaswag",label:"HellaSwag"},{id:"piqa",label:"PIQA"},{id:"arc_easy",label:"ARC-Easy"},{id:"arc_challenge",label:"ARC-Challenge"},{id:"commonsense_qa",label:"CommonsenseQA"},{id:"blimp",label:"BLiMP"}];let r=ee(en),d=ee("all"),l=ee("all"),a=ee("average"),u=ee(null),v=ee([]),b=ee([]),f=ee(0),q=ee([]),S=ee([]),O=ee(!1),R=ee(!1),Z=ee(!1),m=ee("average"),y=ee("desc"),F=ee([]),M=ee([]),U=ee(!1);const _e=10,W=n=>Number(n).toFixed(1),$=n=>Number.isFinite(Number(n))?n>=1e3?`${(n/1e3).toFixed(n>=1e4?1:2)}B`:`${Number(n).toFixed(n<10?2:n<100?1:0)}M`:"—",pe=n=>/^\d{4}-\d{2}-\d{2}$/.test(String(n||""))?n:"—",le=n=>{const o=Number(n);return Number.isFinite(o)&&o>0?o.toLocaleString("en-US"):"—"},re=n=>{const o=Number(n);return!Number.isFinite(o)||o<10?o:Math.round(o/10)*10},ke=n=>(n==null?void 0:n.length)===2?`${W(n[0])}–${W(n[1])}`:"—",ne=n=>{const[o,i]=n.rank_range||[];return o?o===i?`${o}`:`${o}–${i}`:"—"},ye=(n,o=e(a))=>{var i;return o==="average"?n.composite:(i=n.benchmarks)==null?void 0:i[o]},X=(n,o=e(a))=>{var i;return Number((i=ye(n,o))==null?void 0:i.score)},Te=n=>{var o;return((o=s.find(i=>i.id===n))==null?void 0:o.label)||"Average"},Be=n=>{const o=String(n.source||"").trim();return o?/^https?:\/\//i.test(o)?o:`https://huggingface.co/${o}`:null},Ae=(n,o)=>o==="model"?String(n.name||"").toLocaleLowerCase():o==="parameters"?Number(n.params_m):X(n,o);function Re(n){if(e(m)===n){Y(y,e(y)==="asc"?"desc":"asc");return}Y(m,n),Y(y,n==="model"?"asc":"desc")}function ue(n,o,i){const k=i==="asc"?1:-1;return[...n].sort((p,J)=>{const A=Ae(p,o),T=Ae(J,o),P=A===""||typeof A=="number"&&!Number.isFinite(A),L=T===""||typeof T=="number"&&!Number.isFinite(T);return P||L?P===L?0:P?1:-1:typeof A=="number"&&typeof T=="number"?(A-T)*k:String(A).localeCompare(String(T))*k})}function Fe(n,o){const i=[...n].sort((p,J)=>Number(p.params_m)-Number(J.params_m));let k=-1/0;return i.filter(p=>{const J=X(p,o);return!Number.isFinite(J)||J<=k?!1:(k=J,!0)})}function u_(n,o,i){const k=Fe(n,i);if(k.length<2)return null;const p=k.map(z=>({x:o.parameterCoordinate(z.params_m),y:X(z,i)})),J=p.reduce((z,D)=>z+D.x,0)/p.length,A=p.reduce((z,D)=>z+D.y,0)/p.length,T=p.reduce((z,D)=>z+(D.x-J)**2,0),P=T?p.reduce((z,D)=>z+(D.x-J)*(D.y-A),0)/T:0,L=z=>Math.max(o.yMin,Math.min(o.yMax,A+P*(z-J)));return{x1:o.xFromCoordinate(o.xMin),y1:o.y(L(o.xMin)),x2:o.xFromCoordinate(o.xMax),y2:o.y(L(o.xMax))}}function g_(n,o=!1,i="average"){const k=n.map(j=>X(j,i)).filter(Number.isFinite),p=j=>Math.sqrt(Number(j)),J=n.map(j=>p(j.params_m)),A=Math.max(0,Math.floor(Math.min(...J)*10)/10-.1),T=Math.ceil(Math.max(...J)*10)/10+.1,P=Math.min(-5,Math.floor(Math.min(...k)/5)*5-2),L=Math.max(10,Math.ceil(Math.max(...k)/5)*5+2),z=o?820:1120,D=390,ie={left:62,right:28,top:28,bottom:52},ge=j=>ie.left+(j-A)/(T-A||1)*(z-ie.left-ie.right),Pe=j=>ge(p(j)),je=j=>D-ie.bottom-(j-P)/(L-P||1)*(D-ie.top-ie.bottom),Q=Array.from({length:5},(j,B)=>P+B*(L-P)/4),w=Array.from({length:4},(j,B)=>A+B*(T-A)/3);return{width:z,height:D,pad:ie,xMin:A,xMax:T,yMin:P,yMax:L,x:Pe,xFromCoordinate:ge,y:je,yTicks:Q,xTicks:w,parameterCoordinate:p}}function xt(n){const o=n.currentTarget,i=o.closest(".scatter-area").getBoundingClientRect(),k=o.getBoundingClientRect(),p=Number.isFinite(n.clientX)?n.clientX:k.left+k.width/2,J=Number.isFinite(n.clientY)?n.clientY:k.top+k.height/2,A=Math.min(112,Math.max(78,(i.width-20)/2));return{x:Math.max(A+10,Math.min(p-i.left,i.width-A-10)),y:Math.max(J-i.top-16,20)}}function kt(n,o){Y(u,{model:o,...xt(n)})}function h_(n){e(u)&&Y(u,{...e(u),...xt(n)})}B_(()=>{const n=window.matchMedia("(max-width: 760px)"),o=()=>Y(Z,n.matches);return o(),n.addEventListener("change",o),new URLSearchParams(window.location.search).get("demo")==="1"&&Y(r,nn),()=>n.removeEventListener("change",o)}),ze(()=>(e(r),e(d),e(l)),()=>{Y(v,(e(r).models||[]).filter(n=>(e(d)==="all"||n.track===e(d))&&(e(l)==="all"||re(n.params_m)<=Number(e(l)))))}),ze(()=>e(v),()=>{Y(b,e(v).filter(n=>n.complete&&Number.isFinite(Number(n.params_m))))}),ze(()=>(e(b),e(m),e(y)),()=>{Y(F,ue(e(b),e(m),e(y)))}),ze(()=>(e(R),e(F)),()=>{Y(M,e(R)?e(F):e(F).slice(0,_e))}),ze(()=>e(r),()=>{Y(f,(e(r).models||[]).filter(n=>n.complete).length)}),ze(()=>(e(b),e(a)),()=>{Y(q,[...e(b)].filter(n=>Number.isFinite(X(n,e(a)))).sort((n,o)=>X(o,e(a))-X(n,e(a))))}),ze(()=>(e(O),e(q)),()=>{Y(S,e(O)?e(q):e(q).slice(0,_e))}),jo(),Ko();var _t=qn();Mo("1uha8ag",n=>{var o=an();m_(()=>{Co.title="Compact Language Model Index"}),I(n,o)});var ot=x(h(_t),2),Pt=x(h(ot),4),jt=x(h(Pt)),qt=h(jt),b_=h(qt,!0);g(qt),ht(),g(jt),g(Pt),g(ot);var nt=x(ot,2),it=h(nt),Ct=x(h(it),2),Tt=h(Ct),at=h(Tt),st=h(at),lt=h(st);let Et;g(st);var St=x(st,2);we(St,4,()=>[["all","All tracks"],["base","Base"],["instruction","Instruction"]],Oe,(n,o)=>{var i=bt();let k;var p=h(i,!0);g(i),V(()=>{k=oe(i,1,"",null,k,{active:e(d)===o[0]}),N(p,c(()=>o[1]))}),he("click",i,()=>Y(d,o[0])),I(n,i)}),g(St),g(at);var Rt=x(at,2);we(Rt,4,()=>[["all","All"],["10","<10M"],["50","<50M"],["100","<100M"],["300","<300M"]],Oe,(n,o)=>{var i=bt();let k;var p=h(i,!0);g(i),V(()=>{k=oe(i,1,"",null,k,{active:e(l)===o[0]}),N(p,c(()=>o[1]))}),he("click",i,()=>Y(l,o[0])),I(n,i)}),g(Rt),g(Tt),g(Ct),g(it);var rt=x(it,2),ct=h(rt);let zt;var mt=h(ct),Ot=h(mt),De=x(h(Ot));let Jt;var f_=h(De);g(De);var Ke=x(De,2);let At;var y_=h(Ke);g(Ke);var $e=x(Ke,3);let It;var v_=h($e);g($e);var w_=x($e);we(w_,1,()=>c(()=>s.slice(1)),Oe,(n,o)=>{var i=sn();let k;var p=h(i),J=h(p,!0);g(p),g(i),V(()=>{k=oe(i,1,"sortable metric-head",null,k,{sorted:e(m)===e(o).id}),N(J,(e(o),c(()=>e(o).label)))}),he("click",p,()=>Re(e(o).id)),I(n,i)}),g(Ot),g(mt);var Mt=x(mt),x_=h(Mt);{var k_=n=>{var o=ln();I(n,o)},P_=n=>{var o=Q_(),i=Ue(o);we(i,1,()=>e(M),k=>k.id,(k,p)=>{const J=fe(()=>(e(p),c(()=>Be(e(p)))));var A=un(),T=h(A),P=h(T,!0);g(T);var L=x(T),z=h(L);{var D=H=>{var K=rn(),de=h(K),qe=h(de,!0);g(de),g(K),V(()=>{E(K,"href",e(J)),E(K,"aria-label",(e(p),c(()=>`Open ${e(p).name} on Hugging Face`))),N(qe,(e(p),c(()=>e(p).name)))}),I(H,K)},ie=H=>{var K=cn(),de=h(K,!0);g(K),V(()=>N(de,(e(p),c(()=>e(p).name)))),I(H,K)};be(z,H=>{e(J)?H(D):H(ie,-1)})}var ge=x(z,2);{var Pe=H=>{var K=mn(),de=h(K,!0);g(K),V(()=>N(de,(e(p),c(()=>e(p).source)))),I(H,K)};be(ge,H=>{e(p),c(()=>e(p).source)&&H(Pe)})}g(L);var je=x(L),Q=h(je,!0);g(je);var w=x(je),j=h(w,!0);g(w);var B=x(w),G=h(B,!0);g(B);var ce=x(B),me=h(ce);let ve;var Ee=h(me,!0);g(me),g(ce);var ae=x(ce);we(ae,1,()=>s,Oe,(H,K,de)=>{const qe=fe(()=>(e(p),e(K),c(()=>ye(e(p),e(K).id))));var He=pn();oe(He,1,"metric",null,{},{"score-main":de===0});var J_=h(He);{var A_=Je=>{var Ye=dn(),Bt=Ue(Ye,!0),Ft=x(Bt),M_=h(Ft,!0);g(Ft),V((L_,Z_)=>{N(Bt,L_),N(M_,Z_)},[()=>(C(e(qe)),c(()=>W(e(qe).score))),()=>(C(e(qe)),c(()=>ke(e(qe).ci)))]),I(Je,Ye)},I_=Je=>{var Ye=N_("—");I(Je,Ye)};be(J_,Je=>{e(qe)?Je(A_):Je(I_,-1)})}g(He),I(H,He)}),g(A),V((H,K,de,qe)=>{N(P,H),N(Q,K),N(j,de),N(G,qe),ve=oe(me,1,"track",null,ve,{instruction:e(p).track==="instruction"}),N(Ee,(e(p),c(()=>e(p).track==="instruction"?"instruction":"base")))},[()=>(e(p),c(()=>ne(e(p)))),()=>(e(p),c(()=>pe(e(p).release_date))),()=>(e(p),c(()=>$(e(p).params_m))),()=>(e(p),c(()=>le(e(p).vocab_size)))]),Wt(3,A,()=>Yo,()=>({duration:180})),I(k,A)}),I(n,o)};be(x_,n=>{e(b),c(()=>!e(b).length)?n(k_,1):n(P_,-1)})}g(Mt),g(ct),g(rt);var j_=x(rt,2);{var q_=n=>{var o=o_();let i;V(()=>{i=oe(o,1,"bar-toggle table-toggle",null,i,{expanded:e(R)}),E(o,"aria-expanded",e(R)),E(o,"aria-label",(e(R),e(F),c(()=>e(R)?`Show top ${_e} models`:`Show all ${e(F).length} models`))),E(o,"title",(e(R),e(F),c(()=>e(R)?`Show top ${_e} models`:`Show all ${e(F).length} models`)))}),he("click",o,()=>Y(R,!e(R))),I(n,o)};be(j_,n=>{e(F),c(()=>e(F).length>_e)&&n(q_)})}g(nt);var dt=x(nt,2),Lt=x(h(dt),2);we(Lt,5,()=>s,Oe,(n,o)=>{var i=bt();let k;var p=h(i,!0);g(i),V(()=>{k=oe(i,1,"",null,k,{active:e(a)===e(o).id}),N(p,(e(o),c(()=>e(o).label)))}),he("click",i,()=>Y(a,e(o).id)),I(n,i)}),g(Lt),g(dt);var pt=x(dt,2),ut=h(pt),Zt=h(ut),Qt=x(h(Zt),2),C_=h(Qt);g(Qt),g(Zt),ht(2),g(ut);var T_=x(ut,2);{var E_=n=>{const o=fe(()=>e(b)),i=fe(()=>(C(e(o)),e(Z),e(a),c(()=>g_(e(o),e(Z),e(a))))),k=fe(()=>(C(e(o)),C(e(i)),e(a),c(()=>u_(e(o),e(i),e(a)))));var p=vn(),J=h(p),A=h(J),T=x(A);we(T,1,()=>(C(e(i)),c(()=>e(i).yTicks)),Oe,(Q,w)=>{var j=gn(),B=Ue(j),G=x(B),ce=h(G,!0);g(G),V((me,ve,Ee,ae)=>{E(B,"x1",(C(e(i)),c(()=>e(i).pad.left))),E(B,"x2",(C(e(i)),c(()=>e(i).width-e(i).pad.right))),E(B,"y1",me),E(B,"y2",ve),E(G,"x",(C(e(i)),c(()=>e(i).pad.left-14))),E(G,"y",Ee),N(ce,ae)},[()=>(C(e(i)),e(w),c(()=>e(i).y(e(w)))),()=>(C(e(i)),e(w),c(()=>e(i).y(e(w)))),()=>(C(e(i)),e(w),c(()=>e(i).y(e(w))+4)),()=>(e(w),c(()=>W(e(w))))]),I(Q,j)});var P=x(T),L=x(P);we(L,1,()=>(C(e(i)),c(()=>e(i).xTicks)),Oe,(Q,w)=>{var j=hn(),B=h(j,!0);g(j),V((G,ce)=>{E(j,"x",G),E(j,"y",(C(e(i)),c(()=>e(i).height-17))),N(B,ce)},[()=>(C(e(i)),e(w),c(()=>e(i).xFromCoordinate(e(w)))),()=>(e(w),c(()=>$(e(w)**2)))]),I(Q,j)});var z=x(L),D=x(z);{var ie=Q=>{var w=bn();V(()=>{E(w,"x1",(C(e(k)),c(()=>e(k).x1))),E(w,"y1",(C(e(k)),c(()=>e(k).y1))),E(w,"x2",(C(e(k)),c(()=>e(k).x2))),E(w,"y2",(C(e(k)),c(()=>e(k).y2)))}),I(Q,w)};be(D,Q=>{e(k)&&Q(ie)})}var ge=x(D);we(ge,1,()=>e(o),Q=>Q.id,(Q,w)=>{var j=fn();let B;V((G,ce,me)=>{B=oe(j,0,"point",null,B,{"instruction-point":e(w).track==="instruction"}),E(j,"cx",G),E(j,"cy",ce),E(j,"aria-label",me)},[()=>(C(e(i)),e(w),c(()=>e(i).x(e(w).params_m))),()=>(C(e(i)),e(w),e(a),c(()=>e(i).y(X(e(w),e(a))))),()=>(e(w),e(a),c(()=>`${e(w).name}: ${$(e(w).params_m)}, ${Te(e(a))} score ${W(X(e(w),e(a)))}`))]),Ge("mouseenter",j,G=>kt(G,e(w))),he("mousemove",j,h_),Ge("mouseleave",j,()=>Y(u,null)),Ge("focus",j,G=>kt(G,e(w))),Ge("blur",j,()=>Y(u,null)),I(Q,j)}),g(J);var Pe=x(J,2);{var je=Q=>{var w=yn(),j=h(w),B=h(j,!0);g(j);var G=x(j,2),ce=h(G);g(G);var me=x(G,2),ve=h(me),Ee=x(ve),ae=h(Ee);g(Ee),g(me),g(w),V((H,K,de)=>{Xe(w,(e(u),c(()=>`left:${e(u).x}px; top:${e(u).y}px`))),N(B,(e(u),c(()=>e(u).model.name))),N(ce,`${H??""} parameters`),N(ve,`${K??""} `),N(ae,`${de??""} score`)},[()=>(e(u),c(()=>$(e(u).model.params_m))),()=>(e(u),e(a),c(()=>W(X(e(u).model,e(a))))),()=>(e(a),c(()=>Te(e(a))))]),I(Q,w)};be(Pe,Q=>{e(u)&&Q(je)})}g(p),V((Q,w)=>{E(p,"aria-label",Q),E(J,"viewBox",(C(e(i)),c(()=>`0 0 ${e(i).width} ${e(i).height}`))),E(J,"aria-label",w),E(A,"y",(C(e(i)),c(()=>e(i).height/2))),E(A,"transform",(C(e(i)),c(()=>`rotate(-90 7 ${e(i).height/2})`))),E(P,"x1",(C(e(i)),c(()=>e(i).pad.left))),E(P,"x2",(C(e(i)),c(()=>e(i).width-e(i).pad.right))),E(P,"y1",(C(e(i)),c(()=>e(i).height-e(i).pad.bottom))),E(P,"y2",(C(e(i)),c(()=>e(i).height-e(i).pad.bottom))),E(z,"x",(C(e(i)),c(()=>e(i).width/2))),E(z,"y",(C(e(i)),c(()=>e(i).height-1)))},[()=>(e(a),c(()=>`Interactive ${Te(e(a))} score scatter chart`)),()=>(e(a),c(()=>`Scatter chart of model size and chance-normalized ${Te(e(a))} score`))]),I(n,p)},S_=n=>{var o=wn();I(n,o)};be(T_,n=>{e(b),c(()=>e(b).length)?n(E_):n(S_,-1)})}g(pt);var Nt=x(pt,2),R_=x(h(Nt),2);{var z_=n=>{const o=fe(()=>(e(q),e(a),c(()=>Math.min(0,...e(q).flatMap(T=>{var P,L;return[X(T),((L=(P=ye(T,e(a)))==null?void 0:P.ci)==null?void 0:L[0])??X(T)]}))))),i=fe(()=>(e(q),e(a),c(()=>Math.max(10,...e(q).flatMap(T=>{var P,L;return[X(T),((L=(P=ye(T,e(a)))==null?void 0:P.ci)==null?void 0:L[1])??X(T)]})))));var k=Pn(),p=Ue(k);we(p,7,()=>e(S),T=>T.id,(T,P,L)=>{const z=fe(()=>(e(P),c(()=>X(e(P))))),D=fe(()=>(e(P),e(a),c(()=>{var ae;return(ae=ye(e(P),e(a)))==null?void 0:ae.ci}))),ie=fe(()=>(e(z)-e(o))/(e(i)-e(o)||1)*100);var ge=kn(),Pe=h(ge),je=h(Pe,!0);g(Pe);var Q=x(Pe,2),w=h(Q,!0);g(Q);var j=x(Q,2),B=x(h(j));let G;var ce=x(B);{var me=ae=>{var H=xn();let K;V(()=>{K=oe(H,1,"ci-band",null,K,{"instruction-band":e(P).track==="instruction","negative-band":e(z)<0}),Xe(H,(C(e(D)),C(e(o)),C(e(i)),c(()=>`--start:${(e(D)[0]-e(o))/(e(i)-e(o)||1)*100}%; --width:${(e(D)[1]-e(D)[0])/(e(i)-e(o)||1)*100}%`)))}),I(ae,H)};be(ce,ae=>{e(D)&&ae(me)})}g(j);var ve=x(j,2),Ee=h(ve,!0);g(ve),g(ge),V((ae,H)=>{N(je,e(L)+1),E(Q,"title",(e(P),c(()=>e(P).name))),N(w,(e(P),c(()=>e(P).name))),G=oe(B,1,"bar-fill",null,G,{negative:e(z)<0,"instruction-fill":e(P).track==="instruction"}),Xe(B,ae),N(Ee,H)},[()=>(C(e(z)),C(e(ie)),C(e(o)),C(e(i)),c(()=>`--start:${e(z)<0?e(ie):(0-e(o))/(e(i)-e(o)||1)*100}%; --width:${Math.abs(e(z))/(e(i)-e(o)||1)*100}%`)),()=>(C(e(z)),c(()=>W(e(z))))]),Wt(3,ge,()=>Uo,()=>({duration:180})),I(T,ge)}),g(p);var J=x(p,2);{var A=T=>{var P=o_();let L;V(()=>{L=oe(P,1,"bar-toggle",null,L,{expanded:e(O)}),E(P,"aria-expanded",e(O)),E(P,"aria-label",(e(O),e(q),c(()=>e(O)?`Show top ${_e} models`:`Show all ${e(q).length} models`))),E(P,"title",(e(O),e(q),c(()=>e(O)?`Show top ${_e} models`:`Show all ${e(q).length} models`)))}),he("click",P,()=>Y(O,!e(O))),I(T,P)};be(J,T=>{e(q),c(()=>e(q).length>_e)&&T(A)})}V(()=>Xe(p,`--zero:${(0-e(o))/(e(i)-e(o)||1)*100}%`)),I(n,k)},O_=n=>{var o=jn();I(n,o)};be(R_,n=>{e(S),c(()=>e(S).length)?n(z_):n(O_,-1)})}g(Nt),ht(6),g(_t),V(n=>{N(b_,e(f)),E(lt,"aria-pressed",e(U)),Et=oe(lt,1,"",null,Et,{active:e(U)}),zt=oe(ct,1,"",null,zt,{simplified:e(U)}),Jt=oe(De,1,"sortable",null,Jt,{sorted:e(m)==="model"}),At=oe(Ke,1,"sortable",null,At,{sorted:e(m)==="parameters"}),It=oe($e,1,"sortable score-head",null,It,{sorted:e(m)==="average"}),N(C_,`Parameters and ${n??""} score`)},[()=>(e(a),c(()=>Te(e(a))))]),he("click",lt,()=>Y(U,!e(U))),he("click",f_,()=>Re("model")),he("click",y_,()=>Re("parameters")),he("click",v_,()=>Re("average")),I(t,_t),qo()}Eo(["click","mousemove"]);export{On as component};