import{a as J,f as X,t as Ae,c as A_,d as ot}from"../chunks/D0-Lpew9.js";import"../chunks/DP5BbXZE.js";import{o as I_}from"../chunks/Cx7tSjKg.js";import{c as tt,g as o_,aI as n_,h as ne,l as Qe,F as i_,i as M_,L as e,j as L_,S as Z_,k as Kt,n as Ne,f as Le,ab as a_,aj as Q_,a0 as $t,a as N_,aJ as xe,b as _t,aK as B_,s as F_,aC as be,aL as D_,ai as wt,aM as K_,aN as $_,ao as ee,P as Ht,aO as H_,ax as Y_,r as s_,p as l_,aP as ht,O as r_,aQ as U_,aR as G_,ag as V_,m as X_,d as W_,ad as ft,D as eo,E as to,aS as _o,R as oo,ap as c_,t as r,aT as no,aU as io,aV as Ie,a8 as m_,aW as ao,aX as Yt,aY as Ut,aZ as so,a_ as lo,a$ as ro,b0 as co,b1 as mo,b2 as po,q as uo,aE as go,u as Gt,b3 as Vt,b4 as ho,b5 as E,aB as bo,v as fo,al as G,b6 as Oe,b7 as yo,x as U,y as vo,z as h,B as P,A as g,a1 as Ge,b8 as wo,w as Ve}from"../chunks/CdZ30l9o.js";import{a as xo,d as ko,b as ve,s as B,e as Xe}from"../chunks/CryQBVor.js";import{i as he}from"../chunks/DHQwSDIs.js";function Se(t,_){return _}function Po(t,_,s){for(var c=[],p=_.length,l,a=_.length,u=0;u<p;u++){let C=_[u];l_(C,()=>{if(l){if(l.pending.delete(C),l.done.add(C),l.pending.size===0){var R=t.outrogroups;yt(t,wt(l.done)),R.delete(l),R.size===0&&(t.outrogroups=null)}}else a-=1},!1)}if(a===0){var y=c.length===0&&s!==null;if(y){var b=s,f=b.parentNode;V_(f),f.append(b),t.items.clear()}yt(t,_,!y)}else l={pending:new Set(_),done:new Set},(t.outrogroups??(t.outrogroups=new Set)).add(l)}function yt(t,_,s=!0){var c;if(t.pending.size>0){c=new Set;for(const a of t.pending.values())for(const u of a)c.add(t.items.get(u).e)}for(var p=0;p<_.length;p++){var l=_[p];if(c!=null&&c.has(l)){l.f|=xe;const a=document.createDocumentFragment();X_(l,a)}else W_(_[p],s)}}var Xt;function we(t,_,s,c,p,l=null){var a=t,u=new Map,y=(_&n_)!==0;if(y){var b=t;a=ne?Qe(i_(b)):b.appendChild(tt())}ne&&M_();var f=null,C=be(()=>{var M=s();return D_(M)?M:M==null?[]:wt(M)}),R,A=new Map,F=!0;function O(M){(W.effect.f&Y_)===0&&(W.pending.delete(M),W.fallback=f,jo(W,R,a,_,c),f!==null&&(R.length===0?(f.f&xe)===0?s_(f):(f.f^=xe,Ze(f,null,a)):l_(f,()=>{f=null})))}function m(M){W.pending.delete(M)}var v=o_(()=>{R=e(C);var M=R.length;let K=!1;if(ne){var ce=L_(a)===Z_;ce!==(M===0)&&(a=Kt(),Qe(a),Ne(!1),K=!0)}for(var ie=new Set,V=N_,pe=F_(),ae=0;ae<M;ae+=1){ne&&Le.nodeType===a_&&Le.data===Q_&&(a=Le,K=!0,Ne(!1));var se=R[ae],le=c(se,ae),Q=F?null:u.get(le);Q?(Q.v&&$t(Q.v,se),Q.i&&$t(Q.i,ae),pe&&V.unskip_effect(Q.e)):(Q=qo(u,F?a:Xt??(Xt=tt()),se,le,ae,p,_,s),F||(Q.e.f|=xe),u.set(le,Q)),ie.add(le)}if(M===0&&l&&!f&&(F?f=_t(()=>l(a)):(f=_t(()=>l(Xt??(Xt=tt()))),f.f|=xe)),M>ie.size&&B_(),ne&&M>0&&Qe(Kt()),!F)if(A.set(V,ie),pe){for(const[fe,Re]of u)ie.has(fe)||V.skip_effect(Re.e);V.oncommit(O),V.ondiscard(m)}else O(V);K&&Ne(!0),e(C)}),W={effect:v,items:u,pending:A,outrogroups:null,fallback:f};F=!1,ne&&(a=Le)}function Me(t){for(;t!==null&&(t.f&U_)===0;)t=t.next;return t}function jo(t,_,s,c,p){var se,le,Q,fe,Re,Je,Ee,Be,Fe;var l=(c&G_)!==0,a=_.length,u=t.items,y=Me(t.effect.first),b,f=null,C,R=[],A=[],F,O,m,v;if(l)for(v=0;v<a;v+=1)F=_[v],O=p(F,v),m=u.get(O).e,(m.f&xe)===0&&((le=(se=m.nodes)==null?void 0:se.a)==null||le.measure(),(C??(C=new Set)).add(m));for(v=0;v<a;v+=1){if(F=_[v],O=p(F,v),m=u.get(O).e,t.outrogroups!==null)for(const ue of t.outrogroups)ue.pending.delete(m),ue.done.delete(m);if((m.f&ht)!==0&&(s_(m),l&&((fe=(Q=m.nodes)==null?void 0:Q.a)==null||fe.unfix(),(C??(C=new Set)).delete(m))),(m.f&xe)!==0)if(m.f^=xe,m===y)Ze(m,null,s);else{var W=f?f.next:y;m===t.effect.last&&(t.effect.last=m.prev),m.prev&&(m.prev.next=m.next),m.next&&(m.next.prev=m.prev),Te(t,f,m),Te(t,m,W),Ze(m,W,s),f=m,R=[],A=[],y=Me(f.next);continue}if(m!==y){if(b!==void 0&&b.has(m)){if(R.length<A.length){var M=A[0],K;f=M.prev;var ce=R[0],ie=R[R.length-1];for(K=0;K<R.length;K+=1)Ze(R[K],M,s);for(K=0;K<A.length;K+=1)b.delete(A[K]);Te(t,ce.prev,ie.next),Te(t,f,ce),Te(t,ie,M),y=M,f=ie,v-=1,R=[],A=[]}else b.delete(m),Ze(m,y,s),Te(t,m.prev,m.next),Te(t,m,f===null?t.effect.first:f.next),Te(t,f,m),f=m;continue}for(R=[],A=[];y!==null&&y!==m;)(b??(b=new Set)).add(y),A.push(y),y=Me(y.next);if(y===null)continue}(m.f&xe)===0&&R.push(m),f=m,y=Me(m.next)}if(t.outrogroups!==null){for(const ue of t.outrogroups)ue.pending.size===0&&(yt(t,wt(ue.done)),(Re=t.outrogroups)==null||Re.delete(ue));t.outrogroups.size===0&&(t.outrogroups=null)}if(y!==null||b!==void 0){var V=[];if(b!==void 0)for(m of b)(m.f&ht)===0&&V.push(m);for(;y!==null;)(y.f&ht)===0&&y!==t.fallback&&V.push(y),y=Me(y.next);var pe=V.length;if(pe>0){var ae=(c&n_)!==0&&a===0?s:null;if(l){for(v=0;v<pe;v+=1)(Ee=(Je=V[v].nodes)==null?void 0:Je.a)==null||Ee.measure();for(v=0;v<pe;v+=1)(Fe=(Be=V[v].nodes)==null?void 0:Be.a)==null||Fe.fix()}Po(t,V,ae)}}l&&r_(()=>{var ue,De;if(C!==void 0)for(m of C)(De=(ue=m.nodes)==null?void 0:ue.a)==null||De.apply()})}function qo(t,_,s,c,p,l,a,u){var y=(a&K_)!==0?(a&$_)===0?ee(s,!1,!1):Ht(s):null,b=(a&H_)!==0?Ht(p):null;return{v:y,i:b,e:_t(()=>(l(_,y??s,b??p,u),()=>{t.delete(c)}))}}function Ze(t,_,s){if(t.nodes)for(var c=t.nodes.start,p=t.nodes.end,l=_&&(_.f&xe)===0?_.nodes.start:s;c!==null;){var a=ft(c);if(l.before(c),c===p)return;c=a}}function Te(t,_,s){_===null?t.effect.first=s:_.next=s,s===null?t.effect.last=_:s.prev=_}const Co=()=>performance.now(),qe={tick:t=>requestAnimationFrame(t),now:()=>Co(),tasks:new Set};function d_(){const t=qe.now();qe.tasks.forEach(_=>{_.c(t)||(qe.tasks.delete(_),_.f())}),qe.tasks.size!==0&&qe.tick(d_)}function To(t){let _;return qe.tasks.size===0&&qe.tick(d_),{promise:new Promise(s=>{qe.tasks.add(_={c:t,f:s})}),abort(){qe.tasks.delete(_)}}}function We(t,_){m_(()=>{t.dispatchEvent(new CustomEvent(_))})}function Eo(t){if(t==="float")return"cssFloat";if(t==="offset")return"cssOffset";if(t.startsWith("--"))return t;const _=t.split("-");return _.length===1?_[0]:_[0]+_.slice(1).map(s=>s[0].toUpperCase()+s.slice(1)).join("")}function Wt(t){const _={},s=t.split(";");for(const c of s){const[p,l]=c.split(":");if(!p||l===void 0)break;const a=Eo(p.trim());_[a]=l.trim()}return _}const So=t=>t;function Ro(t,_,s,c){var m;var p=(t&no)!==0,l="both",a,u=_.inert,y=_.style.overflow,b,f;function C(){return m_(()=>a??(a=s()(_,(c==null?void 0:c())??{},{direction:l})))}var R={is_global:p,in(){_.inert=u,b=vt(_,C(),f,1,()=>{We(_,"introstart")},()=>{We(_,"introend"),b==null||b.abort(),b=a=void 0,_.style.overflow=y})},out(v){_.inert=!0,f=vt(_,C(),b,0,()=>{We(_,"outrostart")},()=>{We(_,"outroend"),v==null||v()})},stop:()=>{b==null||b.abort(),f==null||f.abort()}},A=eo;if(((m=A.nodes).t??(m.t=[])).push(R),xo){var F=p;if(!F){for(var O=A.parent;O&&(O.f&to)!==0;)for(;(O=O.parent)&&(O.f&_o)===0;);F=!O||(O.f&oo)!==0}F&&c_(()=>{r(()=>R.in())})}}function vt(t,_,s,c,p,l){var a=c===1;if(io(_)){var u,y=!1;return r_(()=>{if(!y){var v=_({direction:a?"in":"out"});u=vt(t,v,s,c,p,l)}}),{abort:()=>{y=!0,u==null||u.abort()},deactivate:()=>u.deactivate(),reset:()=>u.reset(),t:()=>u.t()}}if(s==null||s.deactivate(),!(_!=null&&_.duration)&&!(_!=null&&_.delay))return p(),l(),{abort:Ie,deactivate:Ie,reset:Ie,t:()=>c};const{delay:b=0,css:f,tick:C,easing:R=So}=_;var A=[];if(a&&s===void 0&&(C&&C(0,1),f)){var F=Wt(f(0,1));A.push(F,F)}var O=()=>1-c,m=t.animate(A,{duration:b,fill:"forwards"});return m.onfinish=()=>{m.cancel(),p();var v=(s==null?void 0:s.t())??1-c;s==null||s.abort();var W=c-v,M=_.duration*Math.abs(W),K=[];if(M>0){var ce=!1;if(f)for(var ie=Math.ceil(M/16.666666666666668),V=0;V<=ie;V+=1){var pe=v+W*R(V/ie),ae=Wt(f(pe,1-pe));K.push(ae),ce||(ce=ae.overflow==="hidden")}ce&&(t.style.overflow="hidden"),O=()=>{var se=m.currentTime;return v+W*R(se/M)},C&&To(()=>{if(m.playState!=="running")return!1;var se=O();return C(se,1-se),!0})}m=t.animate(K,{duration:M,fill:"forwards"}),m.onfinish=()=>{O=()=>c,C==null||C(c,1-c),l()}},{abort:()=>{m&&(m.cancel(),m.effect=null,m.onfinish=Ie)},deactivate:()=>{l=Ie},reset:()=>{c===0&&(C==null||C(1,0))},t:()=>O()}}function zo(t,_){let s=null,c=ne;var p;if(ne){s=Le;for(var l=i_(document.head);l!==null&&(l.nodeType!==a_||l.data!==t);)l=ft(l);if(l===null)Ne(!1);else{var a=ft(l);l.remove(),Qe(a)}}ne||(p=document.head.appendChild(tt()));try{o_(()=>{var u=_t(()=>_(p));u.f|=ao})}finally{c&&(Ne(!0),Qe(s))}}const e_=[...` 	
\r\f \v\uFEFF`];function Oo(t,_,s){var c=t==null?"":""+t;if(s){for(var p of Object.keys(s))if(s[p])c=c?c+" "+p:p;else if(c.length)for(var l=p.length,a=0;(a=c.indexOf(p,a))>=0;){var u=a+l;(a===0||e_.includes(c[a-1]))&&(u===c.length||e_.includes(c[u]))?c=(a===0?"":c.substring(0,a))+c.substring(u+1):a=u}}return c===""?null:c}function Jo(t,_){return t==null?null:String(t)}function te(t,_,s,c,p,l){var a=t[Yt];if(ne||a!==s||a===void 0){var u=Oo(s,c,l);(!ne||u!==t.getAttribute("class"))&&(u==null?t.removeAttribute("class"):_?t.className=u:t.setAttribute("class",u)),t[Yt]=s}else if(l&&p!==l)for(var y in l){var b=!!l[y];(p==null||b!==!!p[y])&&t.classList.toggle(y,b)}return l}function et(t,_,s,c){var p=t[Ut];if(ne||p!==_){var l=Jo(_);(!ne||l!==t.getAttribute("style"))&&(l==null?t.removeAttribute("style"):t.style.cssText=l),t[Ut]=_}return c}const Ao=Symbol("is custom element"),Io=Symbol("is html"),Mo=po?"link":"LINK";function S(t,_,s,c){var p=Lo(t);ne&&(p[_]=t.getAttribute(_),_==="src"||_==="srcset"||_==="href"&&t.nodeName===Mo)||p[_]!==(p[_]=s)&&(_==="loading"&&(t[mo]=s),s==null?t.removeAttribute(_):typeof s!="string"&&Zo(t).includes(_)?t[_]=s:t.setAttribute(_,s))}function Lo(t){var _;return t[_=so]??(t[_]={[Ao]:t.nodeName.includes("-"),[Io]:t.namespaceURI===lo})}var t_=new Map;function Zo(t){var _=t.getAttribute("is")||t.nodeName,s=t_.get(_);if(s)return s;t_.set(_,s=[]);for(var c,p=t,l=Element.prototype;l!==p;){c=co(p);for(var a in c)c[a].set&&a!=="innerHTML"&&a!=="textContent"&&a!=="innerText"&&s.push(a);p=ro(p)}return s}function Qo(t=!1){const _=uo,s=_.l.u;if(!s)return;let c=()=>E(_.s);if(t){let p=0,l={};const a=bo(()=>{let u=!1;const y=_.s;for(const b in y)y[b]!==l[b]&&(l[b]=y[b],u=!0);return u&&p++,p});c=()=>e(a)}s.b.length&&go(()=>{__(_,c),Vt(s.b)}),Gt(()=>{const p=r(()=>s.m.map(ho));return()=>{for(const l of p)typeof l=="function"&&l()}}),s.a.length&&Gt(()=>{__(_,c),Vt(s.a)})}function __(t,_){if(t.l.s)for(const s of t.l.s)e(s);_()}function No(t){const _=t-1;return _*_*_+1}function Bo(t,{delay:_=0,duration:s=400,easing:c=No,axis:p="y"}={}){const l=getComputedStyle(t),a=+l.opacity,u=p==="y"?"height":"width",y=parseFloat(l[u]),b=p==="y"?["top","bottom"]:["left","right"],f=b.map(v=>`${v[0].toUpperCase()}${v.slice(1)}`),C=parseFloat(l[`padding${f[0]}`]),R=parseFloat(l[`padding${f[1]}`]),A=parseFloat(l[`margin${f[0]}`]),F=parseFloat(l[`margin${f[1]}`]),O=parseFloat(l[`border${f[0]}Width`]),m=parseFloat(l[`border${f[1]}Width`]);return{delay:_,duration:s,easing:c,css:v=>`overflow: hidden;opacity: ${Math.min(v*20,1)*a};${u}: ${v*y}px;padding-${b[0]}: ${v*C}px;padding-${b[1]}: ${v*R}px;margin-${b[0]}: ${v*A}px;margin-${b[1]}: ${v*F}px;border-${b[0]}-width: ${v*O}px;border-${b[1]}-width: ${v*m}px;min-${u}: 0`}}const Fo=JSON.parse('{"atom-3-4m__gpt-s-1-4m":{"ci":[1.0725360001654316,2.9948782002109477],"difference":2.0326431301418864,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__atom-3-4m":{"ci":[14.14042731298988,16.348961152329448],"difference":15.263255663614302,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__falcon-h1-tiny-r-90m":{"ci":[7.754579096585064,9.903861793761253],"difference":8.820217466378272,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__gpt-s-1-4m":{"ci":[16.129475466747913,18.474038330502612],"difference":17.295898793756187,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__gpt-s2-5m":{"ci":[9.256397771070153,11.38004880060208],"difference":10.331448058039745,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__gpt2-124m":{"ci":[0.7505292033322868,2.7143772000682045],"difference":1.7402333545016724,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__nanowhale-100m-base":{"ci":[10.704834399063397,12.75788043869504],"difference":11.742736810760391,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__pythia-160m":{"ci":[8.298184537762872,10.43661712641525],"difference":9.394030404387307,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__pythia-31m":{"ci":[11.57570162659012,13.766851356448733],"difference":12.659259145773355,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__slm-10m":{"ci":[8.323210002923028,10.392029505159467],"difference":9.360557666936613,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__supra-50m-base":{"ci":[1.3509633161095662,3.2857258513374914],"difference":2.3278594898349447,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"baguettotron__veyra2-apricot-50m-base":{"ci":[2.5296915340167394,4.5001965180924905],"difference":3.547305330072205,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__atom-3-4m":{"ci":[5.420215532883351,7.444210689454811],"difference":6.44303819723603,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__gpt-s-1-4m":{"ci":[7.39930477778452,9.52758940136927],"difference":8.475681327377918,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__gpt-s2-5m":{"ci":[0.5031066010624538,2.4698466105128163],"difference":1.5112305916614743,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__nanowhale-100m-base":{"ci":[1.8741543839787467,3.9548193375016245],"difference":2.922519344382119,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__pythia-160m":{"ci":[-0.580794542209572,1.6693586430525291],"difference":0.5738129380090358,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"falcon-h1-tiny-r-90m__pythia-31m":{"ci":[2.730495758942819,4.922286953231492],"difference":3.8390416793950854,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"falcon-h1-tiny-r-90m__slm-10m":{"ci":[-0.4400500092347234,1.5559306125986496],"difference":0.5403402005583414,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"gemma-3-270m__atom-3-4m":{"ci":[23.385199210048242,25.661827101907907],"difference":24.542681336414624,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__baguettotron":{"ci":[8.231872620662342,10.330917113466759],"difference":9.279425672800324,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__falcon-h1-tiny-r-90m":{"ci":[16.984766782761056,19.19024856397875],"difference":18.099643139178596,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt-s-1-4m":{"ci":[25.31383386993303,27.774047690234024],"difference":26.57532446655651,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt-s2-5m":{"ci":[18.451728069080684,20.73233091916342],"difference":19.61087373084007,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt-x2-125m":{"ci":[2.5100180469723314,4.277907176531071],"difference":3.395722404973594,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__gpt2-124m":{"ci":[10.033805987347693,12.02164981680341],"difference":11.019659027301998,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__mobilellm-r1-140m-base":{"ci":[6.7285212269252535,8.654045161821886],"difference":7.708492950574182,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__nanowhale-100m-base":{"ci":[19.911079186076847,22.136911037973743],"difference":21.022162483560713,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__pythia-160m":{"ci":[17.49071148889552,19.77861550252012],"difference":18.67345607718763,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__pythia-31m":{"ci":[20.72644303179628,23.094161479902546],"difference":21.93868481857368,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__slm-10m":{"ci":[17.546740227831336,19.736908231495857],"difference":18.639983339736936,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__smollm2-135m":{"ci":[0.16836524011662418,1.9651071736191295],"difference":1.0576894080699517,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__supra-50m-base":{"ci":[10.628267365916303,12.551452199235303],"difference":11.60728516263527,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gemma-3-270m__veyra2-apricot-50m-base":{"ci":[11.848671246541473,13.81413489824284],"difference":12.82673100287253,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__atom-3-4m":{"ci":[4.048447096687848,5.816152073030026],"difference":4.931807605574556,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__gpt-s-1-4m":{"ci":[5.994355598920947,7.87200612832699],"difference":6.964450735716443,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__nanowhale-100m-base":{"ci":[0.48907344006903475,2.3424471779865126],"difference":1.4112887527206444,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-s2-5m__pythia-31m":{"ci":[1.3601371630505645,3.270819706973676],"difference":2.3278110877336107,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__atom-3-4m":{"ci":[20.055298011374177,22.273700065241986],"difference":21.146958931441034,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__baguettotron":{"ci":[4.919037209936775,6.853051793908139],"difference":5.883703267826731,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__falcon-h1-tiny-r-90m":{"ci":[13.662735239486574,15.718124447804271],"difference":14.703920734205003,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__gpt-s-1-4m":{"ci":[22.00694789823436,24.304170912361922],"difference":23.17960206158292,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__gpt-s2-5m":{"ci":[15.169372508200368,17.236224767654043],"difference":16.215151325866476,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__gpt2-124m":{"ci":[6.747015411597489,8.522372794247302],"difference":7.623936622328404,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__mobilellm-r1-140m-base":{"ci":[3.4001537890801874,5.2268186213654895],"difference":4.31277054560059,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__nanowhale-100m-base":{"ci":[16.613503795581103,18.6676187860639],"difference":17.626440078587123,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__pythia-160m":{"ci":[14.206915778777304,16.37469514048295],"difference":15.277733672214039,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__pythia-31m":{"ci":[17.423412908889823,19.65878482694102],"difference":18.542962413600087,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__slm-10m":{"ci":[14.224111495050982,16.29911483056891],"difference":15.244260934763345,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__supra-50m-base":{"ci":[7.358523605020158,9.038699445436707],"difference":8.211562757661675,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt-x2-125m__veyra2-apricot-50m-base":{"ci":[8.539745385852653,10.31133646972243],"difference":9.431008597898938,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__atom-3-4m":{"ci":[12.414672706067238,14.51910829641859],"difference":13.523022309112628,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__falcon-h1-tiny-r-90m":{"ci":[6.088666706845199,8.086574541183792],"difference":7.079984111876598,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__gpt-s-1-4m":{"ci":[14.470574653613818,16.628850295397264],"difference":15.555665439254517,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__gpt-s2-5m":{"ci":[7.590743978294489,9.543732770147578],"difference":8.591214703538071,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__nanowhale-100m-base":{"ci":[9.06687024657007,10.940810819228764],"difference":10.002503456258717,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__pythia-160m":{"ci":[6.665984868286377,8.642212289460703],"difference":7.653797049885634,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__pythia-31m":{"ci":[9.944853555192541,11.916576975073232],"difference":10.919025791271682,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__slm-10m":{"ci":[6.635890360218745,8.602630427540772],"difference":7.620324312434939,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"gpt2-124m__supra-50m-base":{"ci":[-0.22231016068905624,1.4204925503246024],"difference":0.5876261353332725,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"gpt2-124m__veyra2-apricot-50m-base":{"ci":[0.9211716297796128,2.705342689004217],"difference":1.8070719755705327,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__atom-3-4m":{"ci":[27.58893756755036,30.042187485115143],"difference":28.789996315485123,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__baguettotron":{"ci":[12.443170128806557,14.63607099576427],"difference":13.526740651870819,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__falcon-h1-tiny-r-90m":{"ci":[21.180647286515022,23.545436928921546],"difference":22.34695811824909,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gemma-3-270m":{"ci":[3.2295084969038235,5.224620472855046],"difference":4.247314979070493,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt-s-1-4m":{"ci":[29.546430185030548,32.048984499243076],"difference":30.822639445627008,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt-s2-5m":{"ci":[22.692921220894892,25.060730091228283],"difference":23.858188709910568,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt-x2-125m":{"ci":[6.543548887485459,8.719161211825703],"difference":7.643037384044087,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__gpt2-124m":{"ci":[14.143222563888733,16.4118593800478],"difference":15.266974006372491,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__mobilellm-r1-140m-base":{"ci":[10.873284350857205,13.02640742452202],"difference":11.955807929644676,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__nanowhale-100m-base":{"ci":[24.094314926003207,26.46428926562647],"difference":25.269477462631208,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__pythia-160m":{"ci":[21.76448313300386,24.12912979103644],"difference":22.920771056258122,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__pythia-31m":{"ci":[24.947883218281085,27.411184938190967],"difference":26.185999797644175,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__qwen2-5-0-5b":{"ci":[-0.7937443045238336,1.2109056554752804],"difference":0.19496537145225148,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"lfm2-350m__slm-10m":{"ci":[21.736389241473272,24.094793643932555],"difference":22.887298318807428,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__smollm2-135m":{"ci":[4.2639180400644365,6.304742227756106],"difference":5.305004387140445,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__supra-50m-base":{"ci":[14.717188310254356,16.936606168548764],"difference":15.854600141705763,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"lfm2-350m__veyra2-apricot-50m-base":{"ci":[15.922087105664474,18.21523002545234],"difference":17.074045981943023,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__atom-3-4m":{"ci":[15.724389468592625,17.9181576661227],"difference":16.83418838584044,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__baguettotron":{"ci":[0.6028456289645157,2.5682754519275246],"difference":1.5709327222261427,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__falcon-h1-tiny-r-90m":{"ci":[9.366761121353298,11.456567965416465],"difference":10.391150188604414,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__gpt-s-1-4m":{"ci":[17.72035213429454,19.952356120653356],"difference":18.86683151598233,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__gpt-s2-5m":{"ci":[10.837860366434668,12.916752286983545],"difference":11.902380780265888,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__gpt2-124m":{"ci":[2.3527040000432247,4.2604573197727165],"difference":3.3111660767278153,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__nanowhale-100m-base":{"ci":[12.244547029232189,14.383753052584806],"difference":13.313669532986532,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__pythia-160m":{"ci":[9.850460315653072,12.110903533239728],"difference":10.96496312661345,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__pythia-31m":{"ci":[13.173488573335227,15.260343800248487],"difference":14.230191867999498,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__slm-10m":{"ci":[9.834008908382621,11.962969758352465],"difference":10.931490389162754,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__supra-50m-base":{"ci":[2.9471642992203297,4.800180307345809],"difference":3.898792212061088,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"mobilellm-r1-140m-base__veyra2-apricot-50m-base":{"ci":[4.204117362859041,6.037751350789785],"difference":5.118238052298348,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"nanowhale-100m-base__atom-3-4m":{"ci":[2.5114800270993958,4.522962101886533],"difference":3.520518852853912,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"nanowhale-100m-base__gpt-s-1-4m":{"ci":[4.520390817955864,6.571451905864101],"difference":5.553161982995798,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"nanowhale-100m-base__pythia-31m":{"ci":[-0.04982526813426013,1.8445162857873265],"difference":0.9165223350129663,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"pythia-160m__atom-3-4m":{"ci":[4.737172915984907,7.018566565651529],"difference":5.8692252592269964,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-160m__gpt-s-1-4m":{"ci":[6.721804247693422,9.058153968300607],"difference":7.901868389368882,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-160m__gpt-s2-5m":{"ci":[-0.19073668035679597,2.031592137727357],"difference":0.9374176536524387,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"pythia-160m__nanowhale-100m-base":{"ci":[1.24315052655882,3.4148330794518693],"difference":2.348706406373083,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-160m__pythia-31m":{"ci":[2.148599543800454,4.357216741831833],"difference":3.2652287413860495,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-31m__atom-3-4m":{"ci":[1.6057485508016338,3.598754829970397],"difference":2.6039965178409457,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"pythia-31m__gpt-s-1-4m":{"ci":[3.553404497040384,5.681548302264185],"difference":4.636639647982832,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__atom-3-4m":{"ci":[27.429065503800224,29.758126578860814],"difference":28.59503094403287,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__baguettotron":{"ci":[12.233474837251677,14.410463289394922],"difference":13.331775280418565,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__falcon-h1-tiny-r-90m":{"ci":[21.042544193912452,23.292755794541097],"difference":22.151992746796836,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gemma-3-270m":{"ci":[3.1289293385506296,4.949670349757808],"difference":4.0523496076182415,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt-s-1-4m":{"ci":[29.358591326483403,31.787910953580926],"difference":30.627674074174756,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt-s2-5m":{"ci":[22.484367687263422,24.80643089248962],"difference":23.663223338458312,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt-x2-125m":{"ci":[6.504515401914428,8.401385534393647],"difference":7.448072012591836,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__gpt2-124m":{"ci":[14.119663769992147,16.08811450435093],"difference":15.07200863492024,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__mobilellm-r1-140m-base":{"ci":[10.797843611952915,12.719193410831675],"difference":11.760842558192424,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__nanowhale-100m-base":{"ci":[23.99988085171323,26.190262920767886],"difference":25.074512091178956,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__pythia-160m":{"ci":[21.56004563074552,23.87905437950645],"difference":22.725805684805874,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__pythia-31m":{"ci":[24.841628881519235,27.14392774955085],"difference":25.991034426191924,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__slm-10m":{"ci":[21.573153354252433,23.791472841951883],"difference":22.69233294735518,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__smollm2-135m":{"ci":[4.182377294645956,6.041752403489344],"difference":5.110039015688194,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__supra-50m-base":{"ci":[14.674719371285,16.64642100282309],"difference":15.659634770253513,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"qwen2-5-0-5b__veyra2-apricot-50m-base":{"ci":[15.843063351497843,17.88305005638959],"difference":16.87908061049077,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__atom-3-4m":{"ci":[5.036334220813094,6.855128264261082],"difference":5.902697996677689,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__gpt-s-1-4m":{"ci":[6.926706396733146,8.94055735587147],"difference":7.935341126819576,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__gpt-s2-5m":{"ci":[0.06659882140217428,1.8222234812887246],"difference":0.970890391103133,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__nanowhale-100m-base":{"ci":[1.4914905145647108,3.2592393440485696],"difference":2.382179143823778,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"slm-10m__pythia-160m":{"ci":[-1.0366806397870105,1.1714324189900007],"difference":0.03347273745069435,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"tie"},"slm-10m__pythia-31m":{"ci":[2.372327463312847,4.223426615282993],"difference":3.2987014788367444,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__atom-3-4m":{"ci":[22.34756564971431,24.615212198542363],"difference":23.484991928344677,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__baguettotron":{"ci":[7.21981046049257,9.250291115821636],"difference":8.221736264730373,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__falcon-h1-tiny-r-90m":{"ci":[15.998703266315285,18.097428607290865],"difference":17.04195373110864,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt-s-1-4m":{"ci":[24.306597878265215,26.660924664410892],"difference":25.517635058486558,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt-s2-5m":{"ci":[17.474818949264733,19.65927462835377],"difference":18.553184322770118,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt-x2-125m":{"ci":[1.520836731078651,3.1770188989552963],"difference":2.338032996903642,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__gpt2-124m":{"ci":[9.01471391018264,10.912959371692486],"difference":9.961969619232047,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__mobilellm-r1-140m-base":{"ci":[5.754466156999673,7.575045374739086],"difference":6.65080354250423,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__nanowhale-100m-base":{"ci":[18.89929834073857,21.076546736195468],"difference":19.964473075490766,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__pythia-160m":{"ci":[16.533556862312427,18.762458693268172],"difference":17.61576666911768,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__pythia-31m":{"ci":[19.743832355893147,21.98849774424981],"difference":20.88099541050373,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__slm-10m":{"ci":[16.48898449494742,18.664225888028522],"difference":17.58229393166699,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__supra-50m-base":{"ci":[9.632456329100759,11.448445329599902],"difference":10.549595754565319,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"smollm2-135m__veyra2-apricot-50m-base":{"ci":[10.828312999027693,12.721441869861339],"difference":11.769041594802578,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__atom-3-4m":{"ci":[11.943975471387882,13.975141551589562],"difference":12.935396173779356,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__falcon-h1-tiny-r-90m":{"ci":[5.4933752538658185,7.494091690632844],"difference":6.4923579765433255,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__gpt-s-1-4m":{"ci":[13.913111223226107,16.026314781815554],"difference":14.968039303921243,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__gpt-s2-5m":{"ci":[7.055711440473382,8.973542914177806],"difference":8.0035885682048,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__nanowhale-100m-base":{"ci":[8.481265325554629,10.332046076912933],"difference":9.414877320925445,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__pythia-160m":{"ci":[5.988387846530205,8.160833376393306],"difference":7.0661709145523615,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__pythia-31m":{"ci":[9.341259784482242,11.317334294187644],"difference":10.33139965593841,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__slm-10m":{"ci":[6.057935337460307,8.006326905907825],"difference":7.032698177101667,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"supra-50m-base__veyra2-apricot-50m-base":{"ci":[0.352626714684538,2.1370426725718357],"difference":1.21944584023726,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__atom-3-4m":{"ci":[10.627668222625106,12.751269936412879],"difference":11.715950333542096,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__falcon-h1-tiny-r-90m":{"ci":[4.269879876321943,6.237640792156269],"difference":5.272912136306066,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__gpt-s-1-4m":{"ci":[12.499782605339313,14.838887566245907],"difference":13.748593463683981,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__gpt-s2-5m":{"ci":[5.8032698087522885,7.724554386452487],"difference":6.784142727967539,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__nanowhale-100m-base":{"ci":[7.226025832871106,9.190466152028234],"difference":8.195431480688185,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__pythia-160m":{"ci":[4.834200955167154,6.909557402281555],"difference":5.846725074315101,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__pythia-31m":{"ci":[8.142263967111687,10.097664917317006],"difference":9.11195381570115,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"},"veyra2-apricot-50m-base__slm-10m":{"ci":[4.863643976660482,6.735987982048942],"difference":5.813252336864407,"matched_samples":{"arc_challenge":1172,"arc_easy":2376,"blimp:Anaphor agreement":2000,"blimp:Argument structure":9000,"blimp:Binding":7000,"blimp:Control / raising":5000,"blimp:Determiner-noun agreement":8000,"blimp:Ellipsis":2000,"blimp:Filler-gap":10000,"blimp:Irregular forms":2000,"blimp:Island effects":8000,"blimp:NPI licensing":7000,"blimp:Quantifiers":4000,"blimp:Subject-verb agreement":6000,"commonsense_qa":1221,"hellaswag":10042,"piqa":1838},"relation":"first_better"}}'),Do="2026-07-20T07:29:29.741057+00:00",Ko=JSON.parse('[{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.39590443686006827,"chance":0.25,"ci":[15.696814562002272,23.321956769055745],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":19.45392491467577},"arc_easy":{"accuracy":0.6641414141414141,"chance":0.25,"ci":[52.637485970819306,57.744107744107744],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":55.21885521885522},"blimp":{"accuracy":0.7906593253968254,"categories":{"Anaphor agreement":0.977,"Argument structure":0.7756666666666666,"Binding":0.7418571428571428,"Control / raising":0.8074,"Determiner-noun agreement":0.9377500000000001,"Ellipsis":0.8654999999999999,"Filler-gap":0.7779999999999999,"Irregular forms":0.728,"Island effects":0.739,"NPI licensing":0.5485714285714286,"Quantifiers":0.7414999999999999,"Subject-verb agreement":0.8476666666666666},"category_count":12,"chance":0.5,"ci":[57.22199222883596,58.42621676587302],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":58.13186507936508},"commonsense_qa":{"accuracy":0.45045045045045046,"chance":0.2,"ci":[27.927927927927925,34.88943488943489],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":31.306306306306304},"hellaswag":{"accuracy":0.4898426608245369,"chance":0.25,"ci":[30.61143198566022,33.25433180641306],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":31.97902144327159},"piqa":{"accuracy":0.6958650707290533,"chance":0.5,"ci":[35.03808487486397,43.2018498367791],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":39.17301414581067}},"complete":true,"composite":{"ci":[42.03623437890757,44.21094708361269],"classification":"reliably above chance","score":43.17876224644195},"id":"lfm2-350m","name":"LFM2 350M","params_m":354.484,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":354483968,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7e746d33d940>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/lfm2-350m","tokenizer":"models/lfm2-350m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[1,2],"release_date":"2025-07-10","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/lfm2-350m/20260716T154647Z-v1.1","source":"LiquidAI/LFM2-350M","track":"instruction","updated_at":"2026-07-16T15:46:57.444261+00:00","vocab_size":65536},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.3199658703071672,"chance":0.25,"ci":[5.802047781569963,13.083048919226394],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":9.328782707622297},"arc_easy":{"accuracy":0.5845959595959596,"chance":0.25,"ci":[41.86307519640854,47.1955667789001],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":44.61279461279461},"blimp":{"accuracy":0.8435170304232805,"categories":{"Anaphor agreement":0.986,"Argument structure":0.8395555555555556,"Binding":0.7834285714285715,"Control / raising":0.8469999999999999,"Determiner-noun agreement":0.95575,"Ellipsis":0.8895,"Filler-gap":0.7902857142857144,"Irregular forms":0.9515,"Island effects":0.694125,"NPI licensing":0.7531428571428572,"Quantifiers":0.7372500000000001,"Subject-verb agreement":0.8946666666666667},"category_count":12,"chance":0.5,"ci":[67.90119460978836,68.90242162698412],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":68.70340608465611},"commonsense_qa":{"accuracy":0.4095004095004095,"chance":0.2,"ci":[22.91154791154791,29.67086404586403],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":26.187551187551186},"hellaswag":{"accuracy":0.5212109141605258,"chance":0.25,"ci":[34.846976033990565,37.50282148310429],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":36.161455221403436},"piqa":{"accuracy":0.6942328618063112,"chance":0.5,"ci":[34.71164309031556,42.763873775843294],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":38.84657236126223}},"complete":true,"composite":{"ci":[41.86348233604306,43.96668012300082],"classification":"reliably above chance","score":42.989729225325554},"id":"qwen2-5-0-5b","name":"Qwen2.5 0.5B","params_m":494,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":494032768,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x72645c9311c0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/qwen2.5-0.5b","tokenizer":"models/qwen2.5-0.5b","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[1,2],"release_date":"2024-09-15","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/qwen2-5-0-5b/20260716T154647Z-v1.1","source":"Qwen/Qwen2.5-0.5B","track":"base","updated_at":"2026-07-16T15:47:02.275688+00:00","vocab_size":151936},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.27986348122866894,"chance":0.25,"ci":[0.5688282138794095,7.281001137656425],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":3.9817974971558585},"arc_easy":{"accuracy":0.5728114478114478,"chance":0.25,"ci":[40.404040404040394,45.62289562289561],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":43.0415263748597},"blimp":{"accuracy":0.8213405423280423,"categories":{"Anaphor agreement":0.99,"Argument structure":0.7935555555555555,"Binding":0.7918571428571427,"Control / raising":0.8374,"Determiner-noun agreement":0.956,"Ellipsis":0.87,"Filler-gap":0.7927142857142858,"Irregular forms":0.9295,"Island effects":0.68875,"NPI licensing":0.668142857142857,"Quantifiers":0.7084999999999999,"Subject-verb agreement":0.8296666666666667},"category_count":12,"chance":0.5,"ci":[63.49301306216931,64.53917559523809],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":64.26810846560846},"commonsense_qa":{"accuracy":0.4258804258804259,"chance":0.2,"ci":[24.754299754299755,31.71580671580671],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":28.235053235053236},"hellaswag":{"accuracy":0.4144592710615415,"chance":0.25,"ci":[20.65325632344155,23.29582420500564],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":21.927902808205534},"piqa":{"accuracy":0.6849836779107725,"chance":0.5,"ci":[32.86180631120783,41.240478781284004],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":36.99673558215451}},"complete":true,"composite":{"ci":[37.774028206553304,39.9138118144519],"classification":"reliably above chance","score":38.912790926162764},"id":"gemma-3-270m","name":"Gemma 3 270M","params_m":270,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"32","batch_sizes":[],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":268098176,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7ff024f928e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gemma-3-270m","tokenizer":"models/gemma-3-270m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[3,3],"release_date":"2025-08-05","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gemma-3-270m/20260716T154647Z-v1.1","source":"google/gemma-3-270m","track":"base","updated_at":"2026-07-16T15:46:49.790812+00:00","vocab_size":262144},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.29692832764505117,"chance":0.25,"ci":[2.6166097838452806,9.670079635949946],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":6.2571103526734895},"arc_easy":{"accuracy":0.5871212121212122,"chance":0.25,"ci":[42.255892255892256,47.53086419753087],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":44.949494949494955},"blimp":{"accuracy":0.8136013227513228,"categories":{"Anaphor agreement":0.977,"Argument structure":0.8337777777777778,"Binding":0.7632857142857142,"Control / raising":0.8392000000000002,"Determiner-noun agreement":0.968,"Ellipsis":0.8745,"Filler-gap":0.7977142857142857,"Irregular forms":0.8380000000000001,"Island effects":0.6892499999999999,"NPI licensing":0.6485714285714286,"Quantifiers":0.67025,"Subject-verb agreement":0.8636666666666667},"category_count":12,"chance":0.5,"ci":[61.932142030423265,63.0667154431217],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":62.72026455026456},"commonsense_qa":{"accuracy":0.35544635544635544,"chance":0.2,"ci":[16.152231777231773,22.81173218673217],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":19.43079443079443},"hellaswag":{"accuracy":0.43218482374029077,"chance":0.25,"ci":[23.016331408086035,25.592511451902016],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":24.291309832038767},"piqa":{"accuracy":0.6817192600652884,"chance":0.5,"ci":[32.097388465723604,40.58759521218715],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":36.34385201305768}},"complete":true,"composite":{"ci":[36.68113506339092,38.849860312348774],"classification":"reliably above chance","score":37.843491883620345},"id":"smollm2-135m","name":"SmolLM2 135M","params_m":135,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":134515008,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7162a1fca3e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/smollm2-135m","tokenizer":"models/smollm2-135m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[4,4],"release_date":"2024-10-31","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/smollm2-135m/20260716T154647Z-v1.1","source":"HuggingFaceTB/SmolLM2-135M","track":"base","updated_at":"2026-07-16T15:47:05.346313+00:00","vocab_size":49152},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.27559726962457337,"chance":0.25,"ci":[-0.11376564277588337,7.053469852104666],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":3.4129692832764498},"arc_easy":{"accuracy":0.5164141414141414,"chance":0.25,"ci":[32.82687991021324,38.21548821548822],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":35.52188552188553},"blimp":{"accuracy":0.8291201719576721,"categories":{"Anaphor agreement":0.9864999999999999,"Argument structure":0.826111111111111,"Binding":0.7557142857142857,"Control / raising":0.8231999999999999,"Determiner-noun agreement":0.9581249999999999,"Ellipsis":0.8534999999999999,"Filler-gap":0.7958571428571429,"Irregular forms":0.923,"Island effects":0.6868749999999999,"NPI licensing":0.7461428571428571,"Quantifiers":0.70725,"Subject-verb agreement":0.8871666666666665},"category_count":12,"chance":0.5,"ci":[64.96420122354498,66.0613258928571],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":65.82403439153441},"commonsense_qa":{"accuracy":0.3464373464373464,"chance":0.2,"ci":[14.926289926289922,21.580671580671574],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":18.3046683046683},"hellaswag":{"accuracy":0.4051981676956781,"chance":0.25,"ci":[19.431388169687313,21.941512314943907],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":20.693089026090416},"piqa":{"accuracy":0.6708378672470077,"chance":0.5,"ci":[29.706202393906423,38.41131664853101],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":34.16757344940153}},"complete":true,"composite":{"ci":[34.34048401085747,36.481411360993164],"classification":"reliably above chance","score":35.52389451013529},"id":"gpt-x2-125m","name":"GPT-X2 125M","params_m":125.081664,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":125081664,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x73557ef813a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gpt-x2-125m","tokenizer":"models/gpt-x2-125m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[5,5],"release_date":"2026-03-25","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-x2-125m/20260716T154647Z-v1.1","source":"AxiomicLabs/GPT-X2-125M","track":"base","updated_at":"2026-07-16T15:46:54.567291+00:00","vocab_size":32768},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/mobilellm-r1-140m-base/20260716T161918Z/models__mobilellm-r1-140m-base/results_2026-07-16T18-27-38.828116.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/mobilellm-r1-140m-base/20260716T161918Z"},"benchmarks":{"arc_challenge":{"accuracy":0.24146757679180889,"chance":0.25,"ci":[-4.550625711035269,2.2753128555176305],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-1.1376564277588153},"arc_easy":{"accuracy":0.49915824915824913,"chance":0.25,"ci":[30.47138047138047,35.746352413019075],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":33.22109988776655},"blimp":{"accuracy":0.8078363425925926,"categories":{"Anaphor agreement":0.9735,"Argument structure":0.7977777777777777,"Binding":0.7627142857142858,"Control / raising":0.8108000000000001,"Determiner-noun agreement":0.9315,"Ellipsis":0.892,"Filler-gap":0.767,"Irregular forms":0.9495,"Island effects":0.6956249999999999,"NPI licensing":0.6222857142857142,"Quantifiers":0.6245,"Subject-verb agreement":0.8668333333333335},"category_count":12,"chance":0.5,"ci":[60.729346891534426,61.79221445105819],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":61.567268518518524},"commonsense_qa":{"accuracy":0.3546273546273546,"chance":0.2,"ci":[15.950040950040947,22.809172809172804],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":19.328419328419326},"hellaswag":{"accuracy":0.3406691894045011,"chance":0.25,"ci":[10.893580296089754,13.297815840138082],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":12.089225253933478},"piqa":{"accuracy":0.6327529923830251,"chance":0.5,"ci":[22.19804134929271,30.794341675734493],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":26.550598476605014}},"complete":true,"composite":{"ci":[30.06941936355877,32.21641211384041],"classification":"reliably above chance","score":31.211043675010572},"id":"mobilellm-r1-140m-base","name":"MobileLLM-R1 140M Base","params_m":140,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":140248512,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7166fab5e020>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/mobilellm-r1-140m-base","tokenizer":"models/mobilellm-r1-140m-base","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[6,6],"release_date":"2025-09-10","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/mobilellm-r1-140m-base/20260716T161918Z","source":"facebook/MobileLLM-R1-140M-base","track":"base","updated_at":"2026-07-16T16:27:44.727364+00:00","vocab_size":128256},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/models__baguettotron/results_2026-07-20T01-21-17.781285.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z"},"benchmarks":{"arc_challenge":{"accuracy":0.3037542662116041,"chance":0.25,"ci":[3.7542662116040924,10.58304891922638],"classification":"reliably above chance","label":"ARC-Challenge","n_samples":1172,"score":7.16723549488055},"arc_easy":{"accuracy":0.5058922558922558,"chance":0.25,"ci":[31.48148148148148,36.700336700336706],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":34.11896745230078},"blimp":{"accuracy":0.7847246362433862,"categories":{"Anaphor agreement":0.8355,"Argument structure":0.7777777777777778,"Binding":0.7462857142857143,"Control / raising":0.7584,"Determiner-noun agreement":0.941,"Ellipsis":0.88,"Filler-gap":0.7588571428571429,"Irregular forms":0.915,"Island effects":0.630375,"NPI licensing":0.623,"Quantifiers":0.7064999999999999,"Subject-verb agreement":0.8439999999999999},"category_count":12,"chance":0.5,"ci":[56.01578158068786,57.203216600529124],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":56.94492724867723},"commonsense_qa":{"accuracy":0.3063063063063063,"chance":0.2,"ci":[10.012285012285009,16.56429156429156],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":13.288288288288284},"hellaswag":{"accuracy":0.35391356303525195,"chance":0.25,"ci":[12.607050388368853,15.050122817499838],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":13.855141738033595},"piqa":{"accuracy":0.6207834602829162,"chance":0.5,"ci":[19.583786724700754,28.726877040261158],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":24.15669205658324}},"complete":true,"composite":{"ci":[28.428847552063168,30.68468269023897],"classification":"reliably above chance","score":29.63151077505659},"id":"baguettotron","name":"Baguettotron","params_m":321,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":"dc3c1cf","inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":320956992,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7ed43ed7b560>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/baguettotron","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[7,7],"release_date":"2025-11-10","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/baguettotron/20260719T231314Z","source":"PleIAs/Baguettotron","track":"instruction","updated_at":"2026-07-19T23:21:23.658293+00:00","vocab_size":65536},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.22440273037542663,"chance":0.25,"ci":[-6.598407281001136,-0.22753128555176305],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.4129692832764498},"arc_easy":{"accuracy":0.3947811447811448,"chance":0.25,"ci":[16.665263748597084,21.885521885521886],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":19.304152637485974},"blimp":{"accuracy":0.8377077380952381,"categories":{"Anaphor agreement":0.997,"Argument structure":0.8356666666666667,"Binding":0.79,"Control / raising":0.8006,"Determiner-noun agreement":0.953875,"Ellipsis":0.848,"Filler-gap":0.8117142857142856,"Irregular forms":0.958,"Island effects":0.693625,"NPI licensing":0.7674285714285715,"Quantifiers":0.7177500000000001,"Subject-verb agreement":0.8788333333333332},"category_count":12,"chance":0.5,"ci":[66.76148528439154,67.78422933201058],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":67.54154761904762},"commonsense_qa":{"accuracy":0.29811629811629814,"chance":0.2,"ci":[9.193284193284189,15.54054054054054],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":12.264537264537264},"hellaswag":{"accuracy":0.31139215295757816,"chance":0.25,"ci":[7.003916882427141,9.380601473809998],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":8.185620394343756},"piqa":{"accuracy":0.6251360174102285,"chance":0.5,"ci":[20.67464635473342,29.37976060935801],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":25.0272034820457}},"complete":true,"composite":{"ci":[26.76217046436712,28.856641117490366],"classification":"reliably above chance","score":27.871165134240197},"id":"gpt2-124m","name":"GPT-2 124M","params_m":124,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false},"model_dtype":"torch.float32","model_num_parameters":124439808,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x707d2cd280e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"float32","pretrained":"models/gpt2-124m","tokenizer":"models/gpt2-124m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[8,9],"release_date":"2022-03-02","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt2-124m/20260716T154647Z-v1.1","source":"openai-community/gpt2","track":"base","updated_at":"2026-07-16T15:46:55.978943+00:00","vocab_size":50257},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.24914675767918087,"chance":0.25,"ci":[-3.526734926052333,3.1854379977246903],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-0.11376564277588337},"arc_easy":{"accuracy":0.4562289562289562,"chance":0.25,"ci":[24.74747474747475,30.1921997755331],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":27.497194163860826},"blimp":{"accuracy":0.7947869047619047,"categories":{"Anaphor agreement":0.913,"Argument structure":0.7863333333333333,"Binding":0.7452857142857142,"Control / raising":0.8018000000000001,"Determiner-noun agreement":0.9412499999999999,"Ellipsis":0.857,"Filler-gap":0.7838571428571429,"Irregular forms":0.8845000000000001,"Island effects":0.63425,"NPI licensing":0.624,"Quantifiers":0.6865,"Subject-verb agreement":0.8796666666666667},"category_count":12,"chance":0.5,"ci":[57.94834143518519,59.13693105158729],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":58.957380952380944},"commonsense_qa":{"accuracy":0.29975429975429974,"chance":0.2,"ci":[9.193284193284189,15.74529074529074],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":12.469287469287464},"hellaswag":{"accuracy":0.31627165903206533,"chance":0.25,"ci":[7.6279625572595116,9.991369581092746],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":8.836221204275377},"piqa":{"accuracy":0.6169749727965179,"chance":0.5,"ci":[18.933623503808494,27.967899891186054],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":23.394994559303584}},"complete":true,"composite":{"ci":[26.15776710993005,28.30631083392145],"classification":"reliably above chance","score":27.338042663290054},"id":"supra-50m-base","name":"Supra 50M Base","params_m":51.78624,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":51786240,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T153917Z/tokenizer_compat","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T153917Z/tokenizer_compat","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7e92921a09a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/supra-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T094052Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[8,9],"release_date":"2026-05-21","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/supra-50m-base/20260716T154647Z-v1.1","source":"SupraLabs/Supra-50M-Base","track":"base","updated_at":"2026-07-16T15:47:07.077007+00:00","vocab_size":32000},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/models__veyra2-apricot-50m-base/results_2026-07-16T18-37-33.567087.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z"},"benchmarks":{"arc_challenge":{"accuracy":0.23464163822525597,"chance":0.25,"ci":[-5.233219567690558,1.137656427758819],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-2.047781569965871},"arc_easy":{"accuracy":0.4276094276094276,"chance":0.25,"ci":[20.93153759820427,26.430976430976433],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":23.681257014590347},"blimp":{"accuracy":0.7935405423280423,"categories":{"Anaphor agreement":0.9115,"Argument structure":0.7822222222222223,"Binding":0.7711428571428571,"Control / raising":0.7988,"Determiner-noun agreement":0.933125,"Ellipsis":0.849,"Filler-gap":0.7371428571428572,"Irregular forms":0.907,"Island effects":0.676375,"NPI licensing":0.6524285714285715,"Quantifiers":0.67225,"Subject-verb agreement":0.8315},"category_count":12,"chance":0.5,"ci":[57.73771858465608,58.93828703703703],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":58.708108465608454},"commonsense_qa":{"accuracy":0.28665028665028663,"chance":0.2,"ci":[7.657657657657656,14.004914004914006],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":10.831285831285827},"hellaswag":{"accuracy":0.3127862975502888,"chance":0.25,"ci":[7.203080395671517,9.540264223594237],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":8.371506340038508},"piqa":{"accuracy":0.6169749727965179,"chance":0.5,"ci":[19.0424374319913,27.856365614798694],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":23.394994559303584}},"complete":true,"composite":{"ci":[24.919591817425154,27.053619924780357],"classification":"reliably above chance","score":26.088907580286257},"id":"veyra2-apricot-50m-base","name":"Veyra2 Apricot 50M Base","params_m":49.3,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"64","batch_sizes":[],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":49303040,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7c9c447b31a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/veyra2-apricot-50m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[10,10],"release_date":"2026-06-14","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/veyra2-apricot-50m-base/20260716T163209Z","source":"veyra-ai/Veyra2-Apricot-50M-Base","track":"base","updated_at":"2026-07-16T16:37:38.898399+00:00","vocab_size":8192},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/falcon-h1-tiny-r-90m/20260720T072155Z/models__falcon-h1-tiny-r-90m/results_2026-07-20T09-28-46.371565.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/falcon-h1-tiny-r-90m/20260720T072155Z"},"benchmarks":{"arc_challenge":{"accuracy":0.24914675767918087,"chance":0.25,"ci":[-3.415813424345847,3.1854379977246903],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-0.11376564277588337},"arc_easy":{"accuracy":0.3926767676767677,"chance":0.25,"ci":[16.273849607182942,21.49410774410773],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":19.023569023569024},"blimp":{"accuracy":0.7126083333333333,"categories":{"Anaphor agreement":0.8009999999999999,"Argument structure":0.6073333333333334,"Binding":0.7232857142857142,"Control / raising":0.6587999999999999,"Determiner-noun agreement":0.8799999999999999,"Ellipsis":0.871,"Filler-gap":0.7327142857142858,"Irregular forms":0.781,"Island effects":0.5905,"NPI licensing":0.704,"Quantifiers":0.44050000000000006,"Subject-verb agreement":0.7611666666666667},"category_count":12,"chance":0.5,"ci":[41.63552711640213,42.98391468253967],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":42.52166666666666},"commonsense_qa":{"accuracy":0.2833742833742834,"chance":0.2,"ci":[7.350532350532346,13.595413595413595],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":10.42178542178542},"hellaswag":{"accuracy":0.30770762796255724,"chance":0.25,"ci":[6.552479585739893,8.876053906924254],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":7.694350395007632},"piqa":{"accuracy":0.6077257889009793,"chance":0.5,"ci":[16.97497279651796,25.897714907508163],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":21.545157780195858}},"complete":true,"composite":{"ci":[19.64362638928377,21.818589193772866],"classification":"reliably above chance","score":20.800441105410517},"id":"falcon-h1-tiny-r-90m","name":"Falcon H1 Tiny R 90M","params_m":90,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":"dc3c1cf","inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":91131072,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7cc583ab0cc0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/falcon-h1-tiny-r-90m","tokenizer":"models/falcon-h1-tiny-r-90m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[11,13],"release_date":"2026-01-12","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/falcon-h1-tiny-r-90m/20260720T072155Z","source":"tiiuae/Falcon-H1-Tiny-R-90M","track":"instruction","updated_at":"2026-07-20T07:28:52.389064+00:00","vocab_size":32768},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.2354948805460751,"chance":0.25,"ci":[-5.1222980659840704,1.251422070534695],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-1.9340159271899877},"arc_easy":{"accuracy":0.3569023569023569,"chance":0.25,"ci":[11.560044893378228,16.835016835016834],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":14.253647586980918},"blimp":{"accuracy":0.772000496031746,"categories":{"Anaphor agreement":0.8959999999999999,"Argument structure":0.7603333333333333,"Binding":0.7562857142857142,"Control / raising":0.757,"Determiner-noun agreement":0.89825,"Ellipsis":0.7855,"Filler-gap":0.7295714285714286,"Irregular forms":0.9505,"Island effects":0.508875,"NPI licensing":0.7078571428571429,"Quantifiers":0.6639999999999999,"Subject-verb agreement":0.8498333333333333},"category_count":12,"chance":0.5,"ci":[53.51157457010582,54.73998561507939],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":54.4000992063492},"commonsense_qa":{"accuracy":0.2596232596232596,"chance":0.2,"ci":[4.484029484029484,10.62653562653563],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":7.452907452907449},"hellaswag":{"accuracy":0.2731527584146584,"chance":0.25,"ci":[1.958109274380932,4.228905264555535],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":3.087034455287789},"piqa":{"accuracy":0.5707290533188248,"chance":0.5,"ci":[9.681719260065288,18.50108813928181],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":14.14581066376497}},"complete":true,"composite":{"ci":[19.071533266309793,21.225442487059897],"classification":"reliably above chance","score":20.26599982402355},"id":"slm-10m","name":"SLM 10M","params_m":9.96864,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":9968640,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T153812Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T153812Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7e9023f9ccc0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/slm-10m","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T101325Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[11,13],"release_date":"2026-06-13","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/slm-10m/20260716T154647Z-v1.1","source":"liodon-ai/slm-10m","track":"base","updated_at":"2026-07-16T15:47:03.814706+00:00","vocab_size":8192},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.23037542662116042,"chance":0.25,"ci":[-5.688282138794084,0.5688282138794095],"classification":"indistinguishable from chance","label":"ARC-Challenge","n_samples":1172,"score":-2.616609783845277},"arc_easy":{"accuracy":0.3686868686868687,"chance":0.25,"ci":[13.18742985409652,18.350168350168353],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":15.824915824915822},"blimp":{"accuracy":0.7286577050264551,"categories":{"Anaphor agreement":0.9245000000000001,"Argument structure":0.7542222222222222,"Binding":0.7448571428571429,"Control / raising":0.7462,"Determiner-noun agreement":0.826125,"Ellipsis":0.759,"Filler-gap":0.6882857142857144,"Irregular forms":0.723,"Island effects":0.631,"NPI licensing":0.5852857142857142,"Quantifiers":0.63075,"Subject-verb agreement":0.7306666666666667},"category_count":12,"chance":0.5,"ci":[44.78496957671956,46.238928736772515],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":45.73154100529102},"commonsense_qa":{"accuracy":0.2932022932022932,"chance":0.2,"ci":[8.37428337428337,14.823914823914825],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":11.650286650286647},"hellaswag":{"accuracy":0.30501892053375823,"chance":0.25,"ci":[6.167430126800774,8.504282015534756],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":7.335856071167764},"piqa":{"accuracy":0.5843307943416758,"chance":0.5,"ci":[12.51360174102285,21.218715995647443],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":16.866158868335155}},"complete":true,"composite":{"ci":[19.111437730092515,21.23647682872686],"classification":"reliably above chance","score":20.2119024553024},"id":"pythia-160m","name":"Pythia 160M","params_m":160,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":162322944,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x71da3d32f9c0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/pythia-160m","tokenizer":"models/pythia-160m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[11,14],"release_date":"2023-02-08","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-160m/20260716T154647Z-v1.1","source":"EleutherAI/pythia-160m","track":"base","updated_at":"2026-07-16T15:46:58.993182+00:00","vocab_size":50304},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.22184300341296928,"chance":0.25,"ci":[-6.939704209328782,-0.6825938566552892],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.754266211604096},"arc_easy":{"accuracy":0.335016835016835,"chance":0.25,"ci":[8.585858585858587,13.916947250280586],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":11.335578002244665},"blimp":{"accuracy":0.7732386243386244,"categories":{"Anaphor agreement":0.9405,"Argument structure":0.7917777777777779,"Binding":0.732857142857143,"Control / raising":0.7488,"Determiner-noun agreement":0.902375,"Ellipsis":0.7815000000000001,"Filler-gap":0.7285714285714285,"Irregular forms":0.915,"Island effects":0.538875,"NPI licensing":0.7108571428571429,"Quantifiers":0.63225,"Subject-verb agreement":0.8555000000000001},"category_count":12,"chance":0.5,"ci":[53.86334275793652,55.039749007936514],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":54.64772486772489},"commonsense_qa":{"accuracy":0.24815724815724816,"chance":0.2,"ci":[3.1505937755937747,9.193284193284189],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":6.019656019656018},"hellaswag":{"accuracy":0.27544313881696875,"chance":0.25,"ci":[2.2638252672110504,4.5083980614751376],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":3.392418508929166},"piqa":{"accuracy":0.5647442872687704,"chance":0.5,"ci":[8.26985854189337,17.41022850924918],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":12.94885745375407}},"complete":true,"composite":{"ci":[18.15426094733613,20.273369388072773],"classification":"reliably above chance","score":19.279532451183385},"id":"gpt-s2-5m","name":"GPT-S2 5M","params_m":5.384,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":5384258,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7891fe910ae0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gpt-s2-5m","tokenizer":"models/gpt-s2-5m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[13,14],"release_date":"2026-06-19","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s2-5m/20260716T154647Z-v1.1","source":"AxiomicLabs/GPT-S2-5M","track":"base","updated_at":"2026-07-16T15:46:53.057636+00:00","vocab_size":4096},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/models__nanowhale-100m-base/results_2026-07-20T01-41-02.427304.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z"},"benchmarks":{"arc_challenge":{"accuracy":0.22440273037542663,"chance":0.25,"ci":[-6.484641638225256,-0.22753128555176305],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.4129692832764498},"arc_easy":{"accuracy":0.3728956228956229,"chance":0.25,"ci":[13.692480359147021,19.023569023569024],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":16.38608305274972},"blimp":{"accuracy":0.7091933531746032,"categories":{"Anaphor agreement":0.794,"Argument structure":0.7473333333333333,"Binding":0.7195714285714286,"Control / raising":0.6965999999999999,"Determiner-noun agreement":0.8567499999999999,"Ellipsis":0.7050000000000001,"Filler-gap":0.6524285714285715,"Irregular forms":0.917,"Island effects":0.495625,"NPI licensing":0.5134285714285715,"Quantifiers":0.64475,"Subject-verb agreement":0.7678333333333334},"category_count":12,"chance":0.5,"ci":[40.99927827380954,42.33725859788362],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":41.83867063492064},"commonsense_qa":{"accuracy":0.27764127764127766,"chance":0.2,"ci":[6.63390663390663,12.878787878787877],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":9.705159705159705},"hellaswag":{"accuracy":0.27126070503883687,"chance":0.25,"ci":[1.63977959237868,3.976631481112662],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":2.8347606718449163},"piqa":{"accuracy":0.573449401523395,"chance":0.5,"ci":[10.228509249183904,19.26006528835691],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":14.689880304678994}},"complete":true,"composite":{"ci":[16.768369462355462,18.88831197015816],"classification":"reliably above chance","score":17.857047764529888},"id":"nanowhale-100m-base","name":"nanowhale 100M Base","params_m":110,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":"dc3c1cf","inference_config":{"batch_size":"32","batch_sizes":[],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true},"model_dtype":"torch.float32","model_num_parameters":110369621,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"torch_seed":1234,"use_cache":null},"n_shot":{"arc_challenge":0,"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa_text":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x75209af931a0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"float32","pretrained":"models/nanowhale-100m-base","tokenizer":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z/tokenizer_compat","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[15,16],"release_date":"2026-04-24","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/nanowhale-100m-base/20260719T232857Z","source":"HuggingFaceTB/nanowhale-100m-base","track":"base","updated_at":"2026-07-19T23:41:08.863409+00:00","vocab_size":129280},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.2167235494880546,"chance":0.25,"ci":[-7.622298065984071,-1.1376564277588153],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-4.436860068259385},"arc_easy":{"accuracy":0.3409090909090909,"chance":0.25,"ci":[9.652076318742987,14.534231200897866],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":12.121212121212118},"blimp":{"accuracy":0.7167433531746031,"categories":{"Anaphor agreement":0.9285,"Argument structure":0.7563333333333333,"Binding":0.7057142857142856,"Control / raising":0.7332,"Determiner-noun agreement":0.857375,"Ellipsis":0.6745,"Filler-gap":0.6648571428571428,"Irregular forms":0.8815,"Island effects":0.535,"NPI licensing":0.45985714285714285,"Quantifiers":0.62375,"Subject-verb agreement":0.7803333333333334},"category_count":12,"chance":0.5,"ci":[42.51403703703704,43.85099421296294],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":43.34867063492063},"commonsense_qa":{"accuracy":0.26453726453726456,"chance":0.2,"ci":[5.098280098280095,11.138411138411136],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":8.067158067158068},"hellaswag":{"accuracy":0.27185819557857,"chance":0.25,"ci":[1.7725552678749275,4.069906393148773],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":2.9144260771426658},"piqa":{"accuracy":0.5680087051142546,"chance":0.5,"ci":[9.140369967355833,18.063112078346034],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":13.601741022850922}},"complete":true,"composite":{"ci":[15.828302485043055,17.965860323148288],"classification":"reliably above chance","score":16.92981203838316},"id":"pythia-31m","name":"Pythia 31M","params_m":31,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false},"model_dtype":"torch.bfloat16","model_num_parameters":30494720,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7ce4241c0180>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/pythia-31m","tokenizer":"models/pythia-31m","trust_remote_code":false,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[15,16],"release_date":"2026-02-24","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/pythia-31m/20260716T154647Z-v1.1","source":"EleutherAI/pythia-31m","track":"base","updated_at":"2026-07-16T15:47:00.508307+00:00","vocab_size":50304},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.21843003412969283,"chance":0.25,"ci":[-7.394766780432309,-1.1376564277588153],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-4.209328782707622},"arc_easy":{"accuracy":0.33080808080808083,"chance":0.25,"ci":[8.361391694725029,13.24354657687991],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":10.774410774410777},"blimp":{"accuracy":0.6819768518518519,"categories":{"Anaphor agreement":0.6659999999999999,"Argument structure":0.7285555555555556,"Binding":0.713857142857143,"Control / raising":0.6869999999999999,"Determiner-noun agreement":0.842125,"Ellipsis":0.708,"Filler-gap":0.6594285714285714,"Irregular forms":0.845,"Island effects":0.5431250000000001,"NPI licensing":0.5187142857142858,"Quantifiers":0.53975,"Subject-verb agreement":0.7321666666666666},"category_count":12,"chance":0.5,"ci":[35.579373181216944,36.99308052248677],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":36.39537037037037},"commonsense_qa":{"accuracy":0.24733824733824733,"chance":0.2,"ci":[2.9484029484029453,9.193284193284189],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":5.917280917280914},"hellaswag":{"accuracy":0.27614021111332404,"chance":0.25,"ci":[2.356768240058423,4.547566885746535],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":3.4853614817765384},"piqa":{"accuracy":0.5571273122959739,"chance":0.5,"ci":[6.855277475516863,15.88683351468989],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":11.42546245919478}},"complete":true,"composite":{"ci":[13.263758832507774,15.316045445100848],"classification":"reliably above chance","score":14.327312373584093},"id":"atom-3-4m","name":"Atom 3.4M","params_m":3.4128,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true},"model_dtype":"torch.float32","model_num_parameters":3412800,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x77ee1d9145e0>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/atom-3-4m","tokenizer":"models/atom-3-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[17,17],"release_date":"2026-06-19","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/atom-3-4m/20260716T154647Z-v1.1","source":"UniversalComputingResearch/Atom3.4m","track":"base","updated_at":"2026-07-16T15:46:48.573093+00:00","vocab_size":4096},{"artifacts":{"result_file":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154647Z-v1.1/results_v1.1_merged.json","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154647Z-v1.1"},"benchmarks":{"arc_challenge":{"accuracy":0.22013651877133106,"chance":0.25,"ci":[-7.053469852104666,-0.6825938566552892],"classification":"below chance","label":"ARC-Challenge","n_samples":1172,"score":-3.9817974971558585},"arc_easy":{"accuracy":0.31776094276094274,"chance":0.25,"ci":[6.509539842873177,11.503928170594838],"classification":"reliably above chance","label":"ARC-Easy","n_samples":2376,"score":9.034792368125698},"blimp":{"accuracy":0.6599056216931216,"categories":{"Anaphor agreement":0.6275,"Argument structure":0.6435555555555555,"Binding":0.7425714285714285,"Control / raising":0.6728000000000001,"Determiner-noun agreement":0.779625,"Ellipsis":0.6525000000000001,"Filler-gap":0.6531428571428571,"Irregular forms":0.8235,"Island effects":0.5603750000000001,"NPI licensing":0.41571428571428576,"Quantifiers":0.6217499999999999,"Subject-verb agreement":0.7258333333333332},"category_count":12,"chance":0.5,"ci":[31.20117526455027,32.64844031084656],"classification":"reliably above chance","label":"BLiMP","n_samples":70000,"score":31.981124338624323},"commonsense_qa":{"accuracy":0.23423423423423423,"chance":0.2,"ci":[1.5125921375921367,7.35309172809171],"classification":"reliably above chance","label":"CommonsenseQA","n_samples":1221,"score":4.279279279279277},"hellaswag":{"accuracy":0.26847241585341564,"chance":0.25,"ci":[1.3075084644493165,3.5919139613622804],"classification":"reliably above chance","label":"HellaSwag","n_samples":10042,"score":2.4629887804554182},"piqa":{"accuracy":0.5516866158868335,"chance":0.5,"ci":[5.658324265505987,14.692600652883549],"classification":"reliably above chance","label":"PIQA","n_samples":1838,"score":10.33732317736671}},"complete":true,"composite":{"ci":[11.210655224075424,13.297313646044007],"classification":"reliably above chance","score":12.287040849624308},"id":"gpt-s-1-4m","name":"GPT-S 1.4M","params_m":1.426,"protocol":{"bootstrap_replicates":2000,"bootstrap_seed":20260716,"harness":"lm-eval==0.4.12","harness_git_hash":null,"inference_config":{"batch_size":"auto","batch_sizes":[64],"bootstrap_iters":100000,"device":"cuda:0","fewshot_seed":1234,"gen_kwargs":{},"limit":null,"model":"hf","model_args":{"dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true},"model_dtype":"torch.bfloat16","model_num_parameters":1426176,"model_revision":"main","model_sha":"","numpy_seed":1234,"random_seed":1234,"tasks":["hellaswag","piqa","arc_easy","arc_challenge","commonsense_qa_text","blimp"],"torch_seed":1234,"use_cache":null,"v1_1_sources":{"rechecked_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154341Z-v1.1/results_v1.1_merged.json","unchanged_tasks_result":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154341Z-v1.1/results_v1.1_merged.json"}},"n_shot":{"arc_easy":0,"blimp":0,"blimp_adjunct_island":0,"blimp_anaphor_gender_agreement":0,"blimp_anaphor_number_agreement":0,"blimp_animate_subject_passive":0,"blimp_animate_subject_trans":0,"blimp_causative":0,"blimp_complex_NP_island":0,"blimp_coordinate_structure_constraint_complex_left_branch":0,"blimp_coordinate_structure_constraint_object_extraction":0,"blimp_determiner_noun_agreement_1":0,"blimp_determiner_noun_agreement_2":0,"blimp_determiner_noun_agreement_irregular_1":0,"blimp_determiner_noun_agreement_irregular_2":0,"blimp_determiner_noun_agreement_with_adj_2":0,"blimp_determiner_noun_agreement_with_adj_irregular_1":0,"blimp_determiner_noun_agreement_with_adj_irregular_2":0,"blimp_determiner_noun_agreement_with_adjective_1":0,"blimp_distractor_agreement_relational_noun":0,"blimp_distractor_agreement_relative_clause":0,"blimp_drop_argument":0,"blimp_ellipsis_n_bar_1":0,"blimp_ellipsis_n_bar_2":0,"blimp_existential_there_object_raising":0,"blimp_existential_there_quantifiers_1":0,"blimp_existential_there_quantifiers_2":0,"blimp_existential_there_subject_raising":0,"blimp_expletive_it_object_raising":0,"blimp_inchoative":0,"blimp_intransitive":0,"blimp_irregular_past_participle_adjectives":0,"blimp_irregular_past_participle_verbs":0,"blimp_irregular_plural_subject_verb_agreement_1":0,"blimp_irregular_plural_subject_verb_agreement_2":0,"blimp_left_branch_island_echo_question":0,"blimp_left_branch_island_simple_question":0,"blimp_matrix_question_npi_licensor_present":0,"blimp_npi_present_1":0,"blimp_npi_present_2":0,"blimp_only_npi_licensor_present":0,"blimp_only_npi_scope":0,"blimp_passive_1":0,"blimp_passive_2":0,"blimp_principle_A_c_command":0,"blimp_principle_A_case_1":0,"blimp_principle_A_case_2":0,"blimp_principle_A_domain_1":0,"blimp_principle_A_domain_2":0,"blimp_principle_A_domain_3":0,"blimp_principle_A_reconstruction":0,"blimp_regular_plural_subject_verb_agreement_1":0,"blimp_regular_plural_subject_verb_agreement_2":0,"blimp_sentential_negation_npi_licensor_present":0,"blimp_sentential_negation_npi_scope":0,"blimp_sentential_subject_island":0,"blimp_superlative_quantifiers_1":0,"blimp_superlative_quantifiers_2":0,"blimp_tough_vs_raising_1":0,"blimp_tough_vs_raising_2":0,"blimp_transitive":0,"blimp_wh_island":0,"blimp_wh_questions_object_gap":0,"blimp_wh_questions_subject_gap":0,"blimp_wh_questions_subject_gap_long_distance":0,"blimp_wh_vs_that_no_gap":0,"blimp_wh_vs_that_no_gap_long_distance":0,"blimp_wh_vs_that_with_gap":0,"blimp_wh_vs_that_with_gap_long_distance":0,"commonsense_qa":0,"hellaswag":0,"piqa":0},"suite":"slm-benchmark-v1.1","task_config":{"arc_challenge":{"dataset_name":"ARC-Challenge","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_challenge.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_challenge","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"arc_easy":{"dataset_name":"ARC-Easy","dataset_path":"allenai/ai2_arc","description":"","doc_to_choice":"{{choices.text}}","doc_to_decontamination_query":"Question: {{question}}\\nAnswer:","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{choices.text}}","doc_to_target":"{{choices.label.index(answerKey)}}","doc_to_text":"Question: {{question}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/arc/arc_easy.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"arc_easy","test_split":"test","training_split":"train","unsafe_code":false,"validation_split":"validation"},"blimp_adjunct_island":{"dataset_name":"adjunct_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/adjunct_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_adjunct_island","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_gender_agreement":{"dataset_name":"anaphor_gender_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_gender_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_gender_agreement","unsafe_code":false,"validation_split":"train"},"blimp_anaphor_number_agreement":{"dataset_name":"anaphor_number_agreement","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/anaphor_number_agreement.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_anaphor_number_agreement","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_passive":{"dataset_name":"animate_subject_passive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_passive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_passive","unsafe_code":false,"validation_split":"train"},"blimp_animate_subject_trans":{"dataset_name":"animate_subject_trans","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/animate_subject_trans.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_animate_subject_trans","unsafe_code":false,"validation_split":"train"},"blimp_causative":{"dataset_name":"causative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/causative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_causative","unsafe_code":false,"validation_split":"train"},"blimp_complex_NP_island":{"dataset_name":"complex_NP_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/complex_NP_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_complex_NP_island","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_complex_left_branch":{"dataset_name":"coordinate_structure_constraint_complex_left_branch","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_complex_left_branch.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_complex_left_branch","unsafe_code":false,"validation_split":"train"},"blimp_coordinate_structure_constraint_object_extraction":{"dataset_name":"coordinate_structure_constraint_object_extraction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/coordinate_structure_constraint_object_extraction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_coordinate_structure_constraint_object_extraction","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_1":{"dataset_name":"determiner_noun_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_2":{"dataset_name":"determiner_noun_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_1":{"dataset_name":"determiner_noun_agreement_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_irregular_2":{"dataset_name":"determiner_noun_agreement_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_2":{"dataset_name":"determiner_noun_agreement_with_adj_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_1":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_1","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adj_irregular_2":{"dataset_name":"determiner_noun_agreement_with_adj_irregular_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adj_irregular_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adj_irregular_2","unsafe_code":false,"validation_split":"train"},"blimp_determiner_noun_agreement_with_adjective_1":{"dataset_name":"determiner_noun_agreement_with_adjective_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/determiner_noun_agreement_with_adjective_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_determiner_noun_agreement_with_adjective_1","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relational_noun":{"dataset_name":"distractor_agreement_relational_noun","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relational_noun.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relational_noun","unsafe_code":false,"validation_split":"train"},"blimp_distractor_agreement_relative_clause":{"dataset_name":"distractor_agreement_relative_clause","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/distractor_agreement_relative_clause.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_distractor_agreement_relative_clause","unsafe_code":false,"validation_split":"train"},"blimp_drop_argument":{"dataset_name":"drop_argument","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/drop_argument.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_drop_argument","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_1":{"dataset_name":"ellipsis_n_bar_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_1","unsafe_code":false,"validation_split":"train"},"blimp_ellipsis_n_bar_2":{"dataset_name":"ellipsis_n_bar_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/ellipsis_n_bar_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_ellipsis_n_bar_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_object_raising":{"dataset_name":"existential_there_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_1":{"dataset_name":"existential_there_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_quantifiers_2":{"dataset_name":"existential_there_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_existential_there_subject_raising":{"dataset_name":"existential_there_subject_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/existential_there_subject_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_existential_there_subject_raising","unsafe_code":false,"validation_split":"train"},"blimp_expletive_it_object_raising":{"dataset_name":"expletive_it_object_raising","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/expletive_it_object_raising.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_expletive_it_object_raising","unsafe_code":false,"validation_split":"train"},"blimp_inchoative":{"dataset_name":"inchoative","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/inchoative.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_inchoative","unsafe_code":false,"validation_split":"train"},"blimp_intransitive":{"dataset_name":"intransitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/intransitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_intransitive","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_adjectives":{"dataset_name":"irregular_past_participle_adjectives","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_adjectives.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_adjectives","unsafe_code":false,"validation_split":"train"},"blimp_irregular_past_participle_verbs":{"dataset_name":"irregular_past_participle_verbs","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_past_participle_verbs.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_past_participle_verbs","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_1":{"dataset_name":"irregular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_irregular_plural_subject_verb_agreement_2":{"dataset_name":"irregular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/irregular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_irregular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_echo_question":{"dataset_name":"left_branch_island_echo_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_echo_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_echo_question","unsafe_code":false,"validation_split":"train"},"blimp_left_branch_island_simple_question":{"dataset_name":"left_branch_island_simple_question","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/left_branch_island_simple_question.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_left_branch_island_simple_question","unsafe_code":false,"validation_split":"train"},"blimp_matrix_question_npi_licensor_present":{"dataset_name":"matrix_question_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/matrix_question_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_matrix_question_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_1":{"dataset_name":"npi_present_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_1","unsafe_code":false,"validation_split":"train"},"blimp_npi_present_2":{"dataset_name":"npi_present_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/npi_present_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_npi_present_2","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_licensor_present":{"dataset_name":"only_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_only_npi_scope":{"dataset_name":"only_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/only_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_only_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_passive_1":{"dataset_name":"passive_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_1","unsafe_code":false,"validation_split":"train"},"blimp_passive_2":{"dataset_name":"passive_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/passive_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_passive_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_c_command":{"dataset_name":"principle_A_c_command","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_c_command.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_c_command","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_1":{"dataset_name":"principle_A_case_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_case_2":{"dataset_name":"principle_A_case_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_case_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_case_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_1":{"dataset_name":"principle_A_domain_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_1","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_2":{"dataset_name":"principle_A_domain_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_2","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_domain_3":{"dataset_name":"principle_A_domain_3","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_domain_3.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_domain_3","unsafe_code":false,"validation_split":"train"},"blimp_principle_A_reconstruction":{"dataset_name":"principle_A_reconstruction","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/principle_A_reconstruction.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_principle_A_reconstruction","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_1":{"dataset_name":"regular_plural_subject_verb_agreement_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_1","unsafe_code":false,"validation_split":"train"},"blimp_regular_plural_subject_verb_agreement_2":{"dataset_name":"regular_plural_subject_verb_agreement_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/regular_plural_subject_verb_agreement_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_regular_plural_subject_verb_agreement_2","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_licensor_present":{"dataset_name":"sentential_negation_npi_licensor_present","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_licensor_present.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_licensor_present","unsafe_code":false,"validation_split":"train"},"blimp_sentential_negation_npi_scope":{"dataset_name":"sentential_negation_npi_scope","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_negation_npi_scope.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_negation_npi_scope","unsafe_code":false,"validation_split":"train"},"blimp_sentential_subject_island":{"dataset_name":"sentential_subject_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/sentential_subject_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_sentential_subject_island","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_1":{"dataset_name":"superlative_quantifiers_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_1","unsafe_code":false,"validation_split":"train"},"blimp_superlative_quantifiers_2":{"dataset_name":"superlative_quantifiers_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/superlative_quantifiers_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_superlative_quantifiers_2","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_1":{"dataset_name":"tough_vs_raising_1","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_1.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_1","unsafe_code":false,"validation_split":"train"},"blimp_tough_vs_raising_2":{"dataset_name":"tough_vs_raising_2","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/tough_vs_raising_2.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_tough_vs_raising_2","unsafe_code":false,"validation_split":"train"},"blimp_transitive":{"dataset_name":"transitive","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/transitive.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_transitive","unsafe_code":false,"validation_split":"train"},"blimp_wh_island":{"dataset_name":"wh_island","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_island.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_island","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_object_gap":{"dataset_name":"wh_questions_object_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_object_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_object_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap":{"dataset_name":"wh_questions_subject_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_questions_subject_gap_long_distance":{"dataset_name":"wh_questions_subject_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_questions_subject_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_questions_subject_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap":{"dataset_name":"wh_vs_that_no_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_no_gap_long_distance":{"dataset_name":"wh_vs_that_no_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_no_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_no_gap_long_distance","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap":{"dataset_name":"wh_vs_that_with_gap","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap","unsafe_code":false,"validation_split":"train"},"blimp_wh_vs_that_with_gap_long_distance":{"dataset_name":"wh_vs_that_with_gap_long_distance","dataset_path":"nyu-mll/blimp","description":"","doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_decontamination_query":"{{sentence_good}} {{sentence_bad}}","doc_to_target":0,"doc_to_text":"","fewshot_config":{"doc_to_choice":"{{[sentence_good, sentence_bad]}}","doc_to_target":0,"doc_to_text":"","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/blimp/wh_vs_that_with_gap_long_distance.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"blimp_wh_vs_that_with_gap_long_distance","unsafe_code":false,"validation_split":"train"},"commonsense_qa_text":{"dataset_path":"tau/commonsense_qa","description":"","doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{ choices.text }}","doc_to_target":"{{ choices.label.index(answerKey) }}","doc_to_text":"Question: {{ question.strip() }}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/config/tasks/commonsense_qa_text.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"commonsense_qa_text","training_split":"train","unsafe_code":false,"validation_split":"validation"},"hellaswag":{"dataset_path":"Rowan/hellaswag","description":"","doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_config":{"doc_to_choice":"choices","doc_to_target":"{{label}}","doc_to_text":"{{query}}","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":"<function process_docs at 0x7c83fdb90c20>","sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/hellaswag/hellaswag.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","process_docs":"def process_docs(dataset: datasets.Dataset) -> datasets.Dataset:\\n    def _process_doc(doc):\\n        ctx = doc[\\"ctx_a\\"] + \\" \\" + doc[\\"ctx_b\\"].capitalize()\\n        out_doc = {\\n            \\"query\\": preprocess(doc[\\"activity_label\\"] + \\": \\" + ctx),\\n            \\"choices\\": [preprocess(ending) for ending in doc[\\"endings\\"]],\\n            \\"gold\\": int(doc[\\"label\\"]),\\n        }\\n        return out_doc\\n\\n    return dataset.map(_process_doc)\\n","repeats":1,"should_decontaminate":false,"target_delimiter":" ","task":"hellaswag","training_split":"train","unsafe_code":false,"validation_split":"validation"},"piqa":{"dataset_path":"baber/piqa","description":"","doc_to_choice":"{{[sol1, sol2]}}","doc_to_decontamination_query":"goal","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_config":{"doc_to_choice":"{{[sol1, sol2]}}","doc_to_target":"label","doc_to_text":"Question: {{goal}}\\nAnswer:","fewshot_delimiter":"\\n\\n","fewshot_indices":null,"gen_prefix":null,"process_docs":null,"sampler":"default","samples":null,"split":null,"target_delimiter":" "},"fewshot_delimiter":"\\n\\n","metadata":{"config_source":"/home/max/PROJECTS/PythonProjects/mcpterm/venv/lib/python3.11/site-packages/lm_eval/tasks/piqa/piqa.yaml","dtype":"bfloat16","pretrained":"models/gpt-s-1-4m","tokenizer":"models/gpt-s-1-4m","trust_remote_code":true,"version":1},"metric_list":[{"aggregation":"mean","higher_is_better":true,"metric":"acc"},{"aggregation":"mean","higher_is_better":true,"metric":"acc_norm"}],"num_fewshot":0,"output_type":"multiple_choice","repeats":1,"should_decontaminate":true,"target_delimiter":" ","task":"piqa","training_split":"train","unsafe_code":false,"validation_split":"validation"}},"task_versions":{"arc_challenge":1,"arc_easy":1,"blimp":"2.0","blimp_adjunct_island":1,"blimp_anaphor_gender_agreement":1,"blimp_anaphor_number_agreement":1,"blimp_animate_subject_passive":1,"blimp_animate_subject_trans":1,"blimp_causative":1,"blimp_complex_NP_island":1,"blimp_coordinate_structure_constraint_complex_left_branch":1,"blimp_coordinate_structure_constraint_object_extraction":1,"blimp_determiner_noun_agreement_1":1,"blimp_determiner_noun_agreement_2":1,"blimp_determiner_noun_agreement_irregular_1":1,"blimp_determiner_noun_agreement_irregular_2":1,"blimp_determiner_noun_agreement_with_adj_2":1,"blimp_determiner_noun_agreement_with_adj_irregular_1":1,"blimp_determiner_noun_agreement_with_adj_irregular_2":1,"blimp_determiner_noun_agreement_with_adjective_1":1,"blimp_distractor_agreement_relational_noun":1,"blimp_distractor_agreement_relative_clause":1,"blimp_drop_argument":1,"blimp_ellipsis_n_bar_1":1,"blimp_ellipsis_n_bar_2":1,"blimp_existential_there_object_raising":1,"blimp_existential_there_quantifiers_1":1,"blimp_existential_there_quantifiers_2":1,"blimp_existential_there_subject_raising":1,"blimp_expletive_it_object_raising":1,"blimp_inchoative":1,"blimp_intransitive":1,"blimp_irregular_past_participle_adjectives":1,"blimp_irregular_past_participle_verbs":1,"blimp_irregular_plural_subject_verb_agreement_1":1,"blimp_irregular_plural_subject_verb_agreement_2":1,"blimp_left_branch_island_echo_question":1,"blimp_left_branch_island_simple_question":1,"blimp_matrix_question_npi_licensor_present":1,"blimp_npi_present_1":1,"blimp_npi_present_2":1,"blimp_only_npi_licensor_present":1,"blimp_only_npi_scope":1,"blimp_passive_1":1,"blimp_passive_2":1,"blimp_principle_A_c_command":1,"blimp_principle_A_case_1":1,"blimp_principle_A_case_2":1,"blimp_principle_A_domain_1":1,"blimp_principle_A_domain_2":1,"blimp_principle_A_domain_3":1,"blimp_principle_A_reconstruction":1,"blimp_regular_plural_subject_verb_agreement_1":1,"blimp_regular_plural_subject_verb_agreement_2":1,"blimp_sentential_negation_npi_licensor_present":1,"blimp_sentential_negation_npi_scope":1,"blimp_sentential_subject_island":1,"blimp_superlative_quantifiers_1":1,"blimp_superlative_quantifiers_2":1,"blimp_tough_vs_raising_1":1,"blimp_tough_vs_raising_2":1,"blimp_transitive":1,"blimp_wh_island":1,"blimp_wh_questions_object_gap":1,"blimp_wh_questions_subject_gap":1,"blimp_wh_questions_subject_gap_long_distance":1,"blimp_wh_vs_that_no_gap":1,"blimp_wh_vs_that_no_gap_long_distance":1,"blimp_wh_vs_that_with_gap":1,"blimp_wh_vs_that_with_gap_long_distance":1,"commonsense_qa_text":"Yaml","hellaswag":1,"piqa":1}},"rank_range":[18,18],"release_date":"2026-06-01","run_dir":"/home/max/PROJECTS/PythonProjects/CompactLMIndex/runs/gpt-s-1-4m/20260716T154647Z-v1.1","source":"AxiomicLabs/GPT-S-1.4M","track":"base","updated_at":"2026-07-16T15:46:51.370873+00:00","vocab_size":4096}]'),$o="slm-benchmark-v1.1",Ho={comparisons:Fo,generated_at:Do,models:Ko,suite:$o},Yo="slm-benchmark-v1.1",Uo="Illustrative data — not an evaluation",Go=[{id:"demo-110m",name:"Illustrative Base 110M",params_m:110,track:"base",complete:!0,rank_range:[1,2],composite:{score:16.8,ci:[15.1,18.6],classification:"reliably above chance"},benchmarks:{hellaswag:{score:12.4,ci:[10.7,14.2]},piqa:{score:20.6,ci:[17.1,24]},arc_easy:{score:15,ci:[11.5,18.5]},arc_challenge:{score:7.1,ci:[3.4,10.7]},commonsense_qa:{score:8.9,ci:[5.2,12.6]},blimp:{score:27.2,ci:[24.8,29.6]}}},{id:"demo-160m",name:"Illustrative Base 160M",params_m:160,track:"base",complete:!0,rank_range:[1,2],composite:{score:17.7,ci:[16,19.5],classification:"reliably above chance"},benchmarks:{hellaswag:{score:13.9,ci:[12.1,15.6]},piqa:{score:22.1,ci:[18.7,25.4]},arc_easy:{score:16.3,ci:[12.9,19.7]},arc_challenge:{score:9.8,ci:[6.1,13.5]},commonsense_qa:{score:10.6,ci:[7.1,14]},blimp:{score:25.8,ci:[23.5,28.1]}}},{id:"demo-60m",name:"Illustrative Base 60M",params_m:60,track:"base",complete:!0,rank_range:[3,3],composite:{score:5.6,ci:[3.9,7.4],classification:"reliably above chance"},benchmarks:{hellaswag:{score:2.1,ci:[.5,3.8]},piqa:{score:7.3,ci:[4,10.5]},arc_easy:{score:3.8,ci:[.8,6.7]},arc_challenge:{score:1.4,ci:[-2.4,5.2]},commonsense_qa:{score:1.2,ci:[-2.1,4.4]},blimp:{score:13.6,ci:[11.2,16]}}}],Vo={suite:Yo,generated_at:Uo,models:Go};var Xo=X('<meta name="description" content="A carefully scored compact language model benchmark from Universal Computing Research."/>'),bt=X("<button> </button>"),Wo=X('<th><button class="sort-button" type="button"><!></button></th>'),en=X('<tr class="empty-row"><td colspan="13">No complete results for this track yet.</td></tr>'),tn=X('<a class="model-link" target="_blank" rel="noreferrer"><span class="model-name"> </span></a>'),_n=X('<span class="model-name"> </span>'),on=X('<span class="model-source"> </span>'),nn=X(" <span> </span>",1),an=X("<td><!></td>"),sn=X('<tr><td class="rank"> </td><td><!> <!></td><td class="metadata"> </td><td> </td><td class="metadata"> </td><td class="track-cell"><span> </span></td><!></tr>'),ln=ot('<line class="grid-line"></line><text class="axis-text" text-anchor="end"> </text>',1),rn=ot('<text class="axis-text" text-anchor="middle"> </text>'),cn=ot('<line class="frontier-line"></line>'),mn=ot('<circle r="7" tabindex="0" role="button"></circle>'),dn=X('<div class="chart-tooltip"><strong> </strong> <span> </span> <b> <em> </em></b></div>'),pn=X('<div class="scatter-area" role="group"><svg preserveAspectRatio="xMidYMid meet" role="img"><text class="axis-title" x="7" text-anchor="middle">Chance-normalized score</text><!><line class="axis-line"></line><!><text class="axis-title" dy="16" text-anchor="middle">Parameters (square-root scale)</text><!><!></svg> <!></div>'),un=X('<div class="empty-chart">Completed evaluations will appear here.</div>'),gn=X("<i></i>"),hn=X('<div class="bar-row"><span class="bar-rank"> </span> <span class="bar-name"> </span> <div class="bar-track"><i class="zero-line"></i><i></i><!></div> <strong> </strong></div>'),bn=X('<button type="button"><span class="bar-toggle-chevron" aria-hidden="true"></span></button>'),fn=X('<div class="bar-list"></div> <!>',1),yn=X('<div class="empty-chart short">No measured values for this selection.</div>'),vn=X('<main><section class="hero" id="top"><div class="hero-copy"><div class="hero-kicker"><p class="eyebrow">Small language model evaluation</p> <span class="edition">Research index</span></div> <h1>Compact<br/>Language Model<br/><span>Index</span></h1> <p class="hero-description">A fixed evaluation of compact language models across six benchmarks, with chance-normalized scores and 95% intervals</p> <a class="hero-link" href="#leaderboard-title">Explore the leaderboard</a></div></section> <section class="summary-grid" aria-label="Suite summary"><article><span class="summary-index">Evaluation breadth</span><div class="summary-value"><strong>6</strong><span>core benchmarks</span></div></article> <article><span class="summary-index">Linguistic depth</span><div class="summary-value"><strong>12</strong><span>BLiMP categories</span></div></article> <article><span class="summary-index">Current sample</span><div class="summary-value"><strong> </strong><span>models measured</span></div></article></section> <section class="leaderboard-section" aria-labelledby="leaderboard-title"><div class="section-heading table-heading"><div><p class="eyebrow">Ranked results</p> <h2 id="leaderboard-title">Core leaderboard</h2></div> <div class="leaderboard-controls"><div class="filter-stack"><div class="top-filter-row"><div class="filters view-filter"><button type="button">Simplified view</button></div> <div class="filters" aria-label="Model track filter"></div></div> <div class="size-filters" aria-label="Maximum parameter count"></div></div></div></div> <div class="table-wrap"><table><thead><tr><th>Rank</th><th><button class="sort-button" type="button">Model</button></th><th class="metadata-head" title="Hugging Face repository creation date">Release date</th><th><button class="sort-button" type="button">Parameters</button></th><th class="metadata-head">Vocab. size</th><th class="track-head">Track</th><th><button class="sort-button" type="button">Score <span>95% CI</span></button></th><!></tr></thead><tbody><!></tbody></table></div> <p class="table-note">ARC-E* = ARC-Easy · ARC-C* = ARC-Challenge · CQA* = CommonsenseQA</p></section> <section class="chart-controls" aria-label="Chart metric selector"><div><p class="eyebrow">Chart metric</p> <p class="section-note">Select a benchmark to update both the scaling chart and score comparison below.</p></div> <div class="task-tabs" aria-label="Benchmark score selector"></div></section> <section class="panel landscape" aria-labelledby="landscape-title"><div class="section-heading"><div><p class="eyebrow">Model scaling</p> <h2 id="landscape-title"> </h2></div> <div class="legend"><span class="legend-dot base"></span>Base <span class="legend-dot instruction"></span>Instruction-tuned <span class="frontier-label"><span class="frontier-key"></span>Pareto trend</span></div></div> <!></section> <section class="panel score-panel" aria-labelledby="scores-title"><div class="section-heading compact-heading"><div><h2 id="scores-title">Model score comparison</h2> <p class="section-note">The top 10 matching models are shown by default. This comparison uses the shared metric selected above.</p></div></div> <!></section> <section class="bottom-grid"><article class="panel method-card"><p class="eyebrow">Scoring method</p> <h3>Chance-normalized composite score</h3> <p class="formula">100 × <span>(accuracy − chance)</span> / <span>(1 − chance)</span></p> <p>For each benchmark, the observed result is normalized against its chance level: chance maps to 0 and a perfect result maps to 100.</p> <p>HellaSwag, PIQA, and CommonsenseQA form a commonsense domain worth 50% of the composite, with each benchmark weighted equally within that domain. Science and world knowledge contribute 25% after combining ARC-Easy (75%) and ARC-Challenge (25%). BLiMP contributes the remaining 25% for grammatical competence and is macro-averaged across its 12 linguistic categories.</p> <p>CommonsenseQA scores the likelihood of each full answer text with length normalization, rather than scoring the A–E answer label. This prevents a model’s letter preference from being mistaken for commonsense capability.</p> <p>Each benchmark interval is a 95% percentile bootstrap. Model comparisons use paired bootstrap resampling on matched evaluation items. Results below chance remain negative, and rank ranges reflect comparisons that do not distinguish neighbouring models.</p></article> <article class="panel protocol-card"><p class="eyebrow">Evaluation protocol</p> <h2>Curated evaluation</h2> <p>CompactLMIndex is a curated benchmark, not an open intake list. We are establishing a Pareto-frontier baseline across model sizes and highlighting training or inference strategies that put a model ahead of its peers at a comparable parameter count.</p> <ul><li>Complete core suite required for a rank</li><li>Zero-shot, non-chat log-likelihood prompts</li><li>Paired bootstrap comparisons on matched questions</li></ul> <p>Every published result is independently checked through our internal verification process. Final inclusion and presentation decisions remain with Universal Computing Research.</p> <p>If you have interesting results for a model you think is suitable for this benchmark, you can open a discussion on the Space for consideration.</p></article></section> <section class="panel additional-card" aria-labelledby="credits-title"><div><p class="eyebrow">Credits &amp; scope</p> <h2 id="credits-title">Efficiency at smaller scales</h2></div> <div class="credits-content"><p>CompactLMIndex looks at smaller language models. At this scale, architecture, training, and inference choices can make a real difference. Every model follows the same evaluation protocol, so results are compared fairly rather than by size alone.</p> <p class="credits-attribution">Inspired by the <a href="https://huggingface.co/open-llm-leaderboard" target="_blank" rel="noreferrer">Open LLM Leaderboard</a>, <a href="https://huggingface.co/spaces/AxiomicLabs/Open_SLM_Leaderboard" target="_blank" rel="noreferrer">Axiomic Labs Open SLM Leaderboard</a>, and <a href="https://github.com/openai/parameter-golf" target="_blank" rel="noreferrer">OpenAI Parameter Golf</a>.</p></div></section> <footer><p>Compact Language Model Index by <a href="https://huggingface.co/UniversalComputingResearch" target="_blank" rel="noreferrer">Universal Computing Research</a></p></footer></main>');function Cn(t,_){fo(_,!1);const s=[{id:"average",label:"Average"},{id:"hellaswag",label:"HellaSwag"},{id:"piqa",label:"PIQA"},{id:"arc_easy",label:"ARC-Easy"},{id:"arc_challenge",label:"ARC-Challenge"},{id:"commonsense_qa",label:"CommonsenseQA"},{id:"blimp",label:"BLiMP"}];let c=ee(Ho),p=ee("all"),l=ee("all"),a=ee("average"),u=ee(null),y=ee([]),b=ee([]),f=ee(0),C=ee([]),R=ee([]),A=ee(!1),F=ee(!1),O=ee("average"),m=ee("desc"),v=ee([]),W=ee(!1);const M=10,K=n=>Number(n).toFixed(1),ce=n=>Number.isFinite(Number(n))?n>=1e3?`${(n/1e3).toFixed(n>=1e4?1:2)}B`:`${Number(n).toFixed(n<10?2:n<100?1:0)}M`:"—",ie=n=>/^\d{4}-\d{2}-\d{2}$/.test(String(n||""))?n:"—",V=n=>{const o=Number(n);return Number.isFinite(o)&&o>0?o.toLocaleString("en-US"):"—"},pe=n=>{const o=Number(n);return!Number.isFinite(o)||o<10?o:Math.round(o/10)*10},ae=n=>(n==null?void 0:n.length)===2?`${K(n[0])}–${K(n[1])}`:"—",se=n=>{const[o,i]=n.rank_range||[];return o?o===i?`${o}`:`${o}–${i}`:"—"},le=(n,o=e(a))=>{var i;return o==="average"?n.composite:(i=n.benchmarks)==null?void 0:i[o]},Q=(n,o=e(a))=>{var i;return Number((i=le(n,o))==null?void 0:i.score)},fe=n=>{var o;return((o=s.find(i=>i.id===n))==null?void 0:o.label)||"Average"},Re=n=>{const o=String(n.source||"").trim();return o?/^https?:\/\//i.test(o)?o:`https://huggingface.co/${o}`:null},Je=(n,o)=>o==="model"?String(n.name||"").toLocaleLowerCase():o==="parameters"?Number(n.params_m):Q(n,o);function Ee(n){if(e(O)===n){G(m,e(m)==="asc"?"desc":"asc");return}G(O,n),G(m,n==="model"?"asc":"desc")}function Be(n,o,i){const x=i==="asc"?1:-1;return[...n].sort((d,L)=>{const I=Je(d,o),T=Je(L,o),k=I===""||typeof I=="number"&&!Number.isFinite(I),Z=T===""||typeof T=="number"&&!Number.isFinite(T);return k||Z?k===Z?0:k?1:-1:typeof I=="number"&&typeof T=="number"?(I-T)*x:String(I).localeCompare(String(T))*x})}function Fe(n,o){const i=[...n].sort((d,L)=>Number(d.params_m)-Number(L.params_m));let x=-1/0;return i.filter(d=>{const L=Q(d,o);return!Number.isFinite(L)||L<=x?!1:(x=L,!0)})}function ue(n,o,i){const x=Fe(n,i);if(x.length<2)return null;const d=x.map(j=>({x:o.parameterCoordinate(j.params_m),y:Q(j,i)})),L=d.reduce((j,z)=>j+z.x,0)/d.length,I=d.reduce((j,z)=>j+z.y,0)/d.length,T=d.reduce((j,z)=>j+(z.x-L)**2,0),k=T?d.reduce((j,z)=>j+(z.x-L)*(z.y-I),0)/T:0,Z=j=>Math.max(o.yMin,Math.min(o.yMax,I+k*(j-L)));return{x1:o.xFromCoordinate(o.xMin),y1:o.y(Z(o.xMin)),x2:o.xFromCoordinate(o.xMax),y2:o.y(Z(o.xMax))}}function De(n,o=!1,i="average"){const x=n.map(q=>Q(q,i)).filter(Number.isFinite),d=q=>Math.sqrt(Number(q)),L=n.map(q=>d(q.params_m)),I=Math.max(0,Math.floor(Math.min(...L)*10)/10-.1),T=Math.ceil(Math.max(...L)*10)/10+.1,k=Math.min(-5,Math.floor(Math.min(...x)/5)*5-2),Z=Math.max(10,Math.ceil(Math.max(...x)/5)*5+2),j=o?820:1120,z=390,_e={left:62,right:28,top:28,bottom:52},ge=q=>_e.left+(q-I)/(T-I||1)*(j-_e.left-_e.right),ke=q=>ge(d(q)),Pe=q=>z-_e.bottom-(q-k)/(Z-k||1)*(z-_e.top-_e.bottom),N=Array.from({length:5},(q,D)=>k+D*(Z-k)/4),w=Array.from({length:4},(q,D)=>I+D*(T-I)/3);return{width:j,height:z,pad:_e,xMin:I,xMax:T,yMin:k,yMax:Z,x:ke,xFromCoordinate:ge,y:Pe,yTicks:N,xTicks:w,parameterCoordinate:d}}function xt(n){const o=n.currentTarget,i=o.closest(".scatter-area").getBoundingClientRect(),x=o.getBoundingClientRect(),d=Number.isFinite(n.clientX)?n.clientX:x.left+x.width/2,L=Number.isFinite(n.clientY)?n.clientY:x.top+x.height/2,I=Math.min(112,Math.max(78,(i.width-20)/2));return{x:Math.max(I+10,Math.min(d-i.left,i.width-I-10)),y:Math.max(L-i.top-16,20)}}function kt(n,o){G(u,{model:o,...xt(n)})}function p_(n){e(u)&&G(u,{...e(u),...xt(n)})}I_(()=>{const n=window.matchMedia("(max-width: 760px)"),o=()=>G(F,n.matches);return o(),n.addEventListener("change",o),new URLSearchParams(window.location.search).get("demo")==="1"&&G(c,Vo),()=>n.removeEventListener("change",o)}),Oe(()=>(e(c),e(p),e(l)),()=>{G(y,(e(c).models||[]).filter(n=>(e(p)==="all"||n.track===e(p))&&(e(l)==="all"||pe(n.params_m)<=Number(e(l)))))}),Oe(()=>e(y),()=>{G(b,e(y).filter(n=>n.complete&&Number.isFinite(Number(n.params_m))))}),Oe(()=>(e(b),e(O),e(m)),()=>{G(v,Be(e(b),e(O),e(m)))}),Oe(()=>e(c),()=>{G(f,(e(c).models||[]).filter(n=>n.complete).length)}),Oe(()=>(e(b),e(a)),()=>{G(C,[...e(b)].filter(n=>Number.isFinite(Q(n,e(a)))).sort((n,o)=>Q(o,e(a))-Q(n,e(a))))}),Oe(()=>(e(A),e(C)),()=>{G(R,e(A)?e(C):e(C).slice(0,M))}),yo(),Qo();var nt=vn();zo("1uha8ag",n=>{var o=Xo();c_(()=>{wo.title="Compact Language Model Index"}),J(n,o)});var it=P(h(nt),2),Pt=P(h(it),4),jt=P(h(Pt)),qt=h(jt),u_=h(qt,!0);g(qt),Ge(),g(jt),g(Pt),g(it);var at=P(it,2),st=h(at),Ct=P(h(st),2),Tt=h(Ct),lt=h(Tt),rt=h(lt),ct=h(rt);let Et;g(rt);var St=P(rt,2);we(St,4,()=>[["all","All tracks"],["base","Base"],["instruction","Instruction"]],Se,(n,o)=>{var i=bt();let x;var d=h(i,!0);g(i),U(()=>{x=te(i,1,"",null,x,{active:e(p)===o[0]}),B(d,r(()=>o[1]))}),ve("click",i,()=>G(p,o[0])),J(n,i)}),g(St),g(lt);var Rt=P(lt,2);we(Rt,4,()=>[["all","All"],["10","<10M"],["50","<50M"],["100","<100M"],["300","<300M"]],Se,(n,o)=>{var i=bt();let x;var d=h(i,!0);g(i),U(()=>{x=te(i,1,"",null,x,{active:e(l)===o[0]}),B(d,r(()=>o[1]))}),ve("click",i,()=>G(l,o[0])),J(n,i)}),g(Rt),g(Tt),g(Ct),g(st);var zt=P(st,2),mt=h(zt);let Ot;var dt=h(mt),Jt=h(dt),Ke=P(h(Jt));let At;var g_=h(Ke);g(Ke);var $e=P(Ke,2);let It;var h_=h($e);g($e);var He=P($e,3);let Mt;var b_=h(He);g(He);var f_=P(He);we(f_,1,()=>r(()=>s.slice(1)),Se,(n,o)=>{var i=Wo();let x;var d=h(i),L=h(d);{var I=j=>{var z=Ae("ARC-E*");J(j,z)},T=j=>{var z=Ae("ARC-C*");J(j,z)},k=j=>{var z=Ae("CQA*");J(j,z)},Z=j=>{var z=Ae();U(()=>B(z,(e(o),r(()=>e(o).label)))),J(j,z)};he(L,j=>{e(o),r(()=>e(o).id==="arc_easy")?j(I):(e(o),r(()=>e(o).id==="arc_challenge")?j(T,1):(e(o),r(()=>e(o).id==="commonsense_qa")?j(k,2):j(Z,-1)))})}g(d),g(i),U(()=>{x=te(i,1,"sortable metric-head",null,x,{sorted:e(O)===e(o).id}),S(d,"aria-label",(e(o),r(()=>`Sort by ${e(o).label}`)))}),ve("click",d,()=>Ee(e(o).id)),J(n,i)}),g(Jt),g(dt);var Lt=P(dt),y_=h(Lt);{var v_=n=>{var o=en();J(n,o)},w_=n=>{var o=A_(),i=Ve(o);we(i,1,()=>e(v),x=>x.id,(x,d)=>{const L=be(()=>(e(d),r(()=>Re(e(d)))));var I=sn(),T=h(I),k=h(T,!0);g(T);var Z=P(T),j=h(Z);{var z=H=>{var $=tn(),de=h($),je=h(de,!0);g(de),g($),U(()=>{S($,"href",e(L)),S($,"aria-label",(e(d),r(()=>`Open ${e(d).name} on Hugging Face`))),B(je,(e(d),r(()=>e(d).name)))}),J(H,$)},_e=H=>{var $=_n(),de=h($,!0);g($),U(()=>B(de,(e(d),r(()=>e(d).name)))),J(H,$)};he(j,H=>{e(L)?H(z):H(_e,-1)})}var ge=P(j,2);{var ke=H=>{var $=on(),de=h($,!0);g($),U(()=>B(de,(e(d),r(()=>e(d).source)))),J(H,$)};he(ge,H=>{e(d),r(()=>e(d).source)&&H(ke)})}g(Z);var Pe=P(Z),N=h(Pe,!0);g(Pe);var w=P(Pe),q=h(w,!0);g(w);var D=P(w),Y=h(D,!0);g(D);var re=P(D),me=h(re);let ye;var Ce=h(me,!0);g(me),g(re);var oe=P(re);we(oe,1,()=>s,Se,(H,$,de)=>{const je=be(()=>(e(d),e($),r(()=>le(e(d),e($).id))));var Ye=an();te(Ye,1,"metric",null,{},{"score-main":de===0});var E_=h(Ye);{var S_=ze=>{var Ue=nn(),Ft=Ve(Ue,!0),Dt=P(Ft),z_=h(Dt,!0);g(Dt),U((O_,J_)=>{B(Ft,O_),B(z_,J_)},[()=>(E(e(je)),r(()=>K(e(je).score))),()=>(E(e(je)),r(()=>ae(e(je).ci)))]),J(ze,Ue)},R_=ze=>{var Ue=Ae("—");J(ze,Ue)};he(E_,ze=>{e(je)?ze(S_):ze(R_,-1)})}g(Ye),J(H,Ye)}),g(I),U((H,$,de,je)=>{B(k,H),B(N,$),B(q,de),B(Y,je),ye=te(me,1,"track",null,ye,{instruction:e(d).track==="instruction"}),B(Ce,(e(d),r(()=>e(d).track==="instruction"?"instruction":"base")))},[()=>(e(d),r(()=>se(e(d)))),()=>(e(d),r(()=>ie(e(d).release_date))),()=>(e(d),r(()=>ce(e(d).params_m))),()=>(e(d),r(()=>V(e(d).vocab_size)))]),J(x,I)}),J(n,o)};he(y_,n=>{e(b),r(()=>!e(b).length)?n(v_,1):n(w_,-1)})}g(Lt),g(mt),g(zt),Ge(2),g(at);var pt=P(at,2),Zt=P(h(pt),2);we(Zt,5,()=>s,Se,(n,o)=>{var i=bt();let x;var d=h(i,!0);g(i),U(()=>{x=te(i,1,"",null,x,{active:e(a)===e(o).id}),B(d,(e(o),r(()=>e(o).label)))}),ve("click",i,()=>G(a,e(o).id)),J(n,i)}),g(Zt),g(pt);var ut=P(pt,2),gt=h(ut),Qt=h(gt),Nt=P(h(Qt),2),x_=h(Nt);g(Nt),g(Qt),Ge(2),g(gt);var k_=P(gt,2);{var P_=n=>{const o=be(()=>e(b)),i=be(()=>(E(e(o)),e(F),e(a),r(()=>De(e(o),e(F),e(a))))),x=be(()=>(E(e(o)),E(e(i)),e(a),r(()=>ue(e(o),e(i),e(a)))));var d=pn(),L=h(d),I=h(L),T=P(I);we(T,1,()=>(E(e(i)),r(()=>e(i).yTicks)),Se,(N,w)=>{var q=ln(),D=Ve(q),Y=P(D),re=h(Y,!0);g(Y),U((me,ye,Ce,oe)=>{S(D,"x1",(E(e(i)),r(()=>e(i).pad.left))),S(D,"x2",(E(e(i)),r(()=>e(i).width-e(i).pad.right))),S(D,"y1",me),S(D,"y2",ye),S(Y,"x",(E(e(i)),r(()=>e(i).pad.left-14))),S(Y,"y",Ce),B(re,oe)},[()=>(E(e(i)),e(w),r(()=>e(i).y(e(w)))),()=>(E(e(i)),e(w),r(()=>e(i).y(e(w)))),()=>(E(e(i)),e(w),r(()=>e(i).y(e(w))+4)),()=>(e(w),r(()=>K(e(w))))]),J(N,q)});var k=P(T),Z=P(k);we(Z,1,()=>(E(e(i)),r(()=>e(i).xTicks)),Se,(N,w)=>{var q=rn(),D=h(q,!0);g(q),U((Y,re)=>{S(q,"x",Y),S(q,"y",(E(e(i)),r(()=>e(i).height-17))),B(D,re)},[()=>(E(e(i)),e(w),r(()=>e(i).xFromCoordinate(e(w)))),()=>(e(w),r(()=>ce(e(w)**2)))]),J(N,q)});var j=P(Z),z=P(j);{var _e=N=>{var w=cn();U(()=>{S(w,"x1",(E(e(x)),r(()=>e(x).x1))),S(w,"y1",(E(e(x)),r(()=>e(x).y1))),S(w,"x2",(E(e(x)),r(()=>e(x).x2))),S(w,"y2",(E(e(x)),r(()=>e(x).y2)))}),J(N,w)};he(z,N=>{e(x)&&N(_e)})}var ge=P(z);we(ge,1,()=>e(o),N=>N.id,(N,w)=>{var q=mn();let D;U((Y,re,me)=>{D=te(q,0,"point",null,D,{"instruction-point":e(w).track==="instruction"}),S(q,"cx",Y),S(q,"cy",re),S(q,"aria-label",me)},[()=>(E(e(i)),e(w),r(()=>e(i).x(e(w).params_m))),()=>(E(e(i)),e(w),e(a),r(()=>e(i).y(Q(e(w),e(a))))),()=>(e(w),e(a),r(()=>`${e(w).name}: ${ce(e(w).params_m)}, ${fe(e(a))} score ${K(Q(e(w),e(a)))}`))]),Xe("mouseenter",q,Y=>kt(Y,e(w))),ve("mousemove",q,p_),Xe("mouseleave",q,()=>G(u,null)),Xe("focus",q,Y=>kt(Y,e(w))),Xe("blur",q,()=>G(u,null)),J(N,q)}),g(L);var ke=P(L,2);{var Pe=N=>{var w=dn(),q=h(w),D=h(q,!0);g(q);var Y=P(q,2),re=h(Y);g(Y);var me=P(Y,2),ye=h(me),Ce=P(ye),oe=h(Ce);g(Ce),g(me),g(w),U((H,$,de)=>{et(w,(e(u),r(()=>`left:${e(u).x}px; top:${e(u).y}px`))),B(D,(e(u),r(()=>e(u).model.name))),B(re,`${H??""} parameters`),B(ye,`${$??""} `),B(oe,`${de??""} score`)},[()=>(e(u),r(()=>ce(e(u).model.params_m))),()=>(e(u),e(a),r(()=>K(Q(e(u).model,e(a))))),()=>(e(a),r(()=>fe(e(a))))]),J(N,w)};he(ke,N=>{e(u)&&N(Pe)})}g(d),U((N,w)=>{S(d,"aria-label",N),S(L,"viewBox",(E(e(i)),r(()=>`0 0 ${e(i).width} ${e(i).height}`))),S(L,"aria-label",w),S(I,"y",(E(e(i)),r(()=>e(i).height/2))),S(I,"transform",(E(e(i)),r(()=>`rotate(-90 7 ${e(i).height/2})`))),S(k,"x1",(E(e(i)),r(()=>e(i).pad.left))),S(k,"x2",(E(e(i)),r(()=>e(i).width-e(i).pad.right))),S(k,"y1",(E(e(i)),r(()=>e(i).height-e(i).pad.bottom))),S(k,"y2",(E(e(i)),r(()=>e(i).height-e(i).pad.bottom))),S(j,"x",(E(e(i)),r(()=>e(i).width/2))),S(j,"y",(E(e(i)),r(()=>e(i).height-1)))},[()=>(e(a),r(()=>`Interactive ${fe(e(a))} score scatter chart`)),()=>(e(a),r(()=>`Scatter chart of model size and chance-normalized ${fe(e(a))} score`))]),J(n,d)},j_=n=>{var o=un();J(n,o)};he(k_,n=>{e(b),r(()=>e(b).length)?n(P_):n(j_,-1)})}g(ut);var Bt=P(ut,2),q_=P(h(Bt),2);{var C_=n=>{const o=be(()=>(e(C),e(a),r(()=>Math.min(0,...e(C).flatMap(T=>{var k,Z;return[Q(T),((Z=(k=le(T,e(a)))==null?void 0:k.ci)==null?void 0:Z[0])??Q(T)]}))))),i=be(()=>(e(C),e(a),r(()=>Math.max(10,...e(C).flatMap(T=>{var k,Z;return[Q(T),((Z=(k=le(T,e(a)))==null?void 0:k.ci)==null?void 0:Z[1])??Q(T)]})))));var x=fn(),d=Ve(x);we(d,7,()=>e(R),T=>T.id,(T,k,Z)=>{const j=be(()=>(e(k),r(()=>Q(e(k))))),z=be(()=>(e(k),e(a),r(()=>{var oe;return(oe=le(e(k),e(a)))==null?void 0:oe.ci}))),_e=be(()=>(e(j)-e(o))/(e(i)-e(o)||1)*100);var ge=hn(),ke=h(ge),Pe=h(ke,!0);g(ke);var N=P(ke,2),w=h(N,!0);g(N);var q=P(N,2),D=P(h(q));let Y;var re=P(D);{var me=oe=>{var H=gn();let $;U(()=>{$=te(H,1,"ci-band",null,$,{"instruction-band":e(k).track==="instruction","negative-band":e(j)<0}),et(H,(E(e(z)),E(e(o)),E(e(i)),r(()=>`--start:${(e(z)[0]-e(o))/(e(i)-e(o)||1)*100}%; --width:${(e(z)[1]-e(z)[0])/(e(i)-e(o)||1)*100}%`)))}),J(oe,H)};he(re,oe=>{e(z)&&oe(me)})}g(q);var ye=P(q,2),Ce=h(ye,!0);g(ye),g(ge),U((oe,H)=>{B(Pe,e(Z)+1),S(N,"title",(e(k),r(()=>e(k).name))),B(w,(e(k),r(()=>e(k).name))),Y=te(D,1,"bar-fill",null,Y,{negative:e(j)<0,"instruction-fill":e(k).track==="instruction"}),et(D,oe),B(Ce,H)},[()=>(E(e(j)),E(e(_e)),E(e(o)),E(e(i)),r(()=>`--start:${e(j)<0?e(_e):(0-e(o))/(e(i)-e(o)||1)*100}%; --width:${Math.abs(e(j))/(e(i)-e(o)||1)*100}%`)),()=>(E(e(j)),r(()=>K(e(j))))]),Ro(3,ge,()=>Bo,()=>({duration:180})),J(T,ge)}),g(d);var L=P(d,2);{var I=T=>{var k=bn();let Z;U(()=>{Z=te(k,1,"bar-toggle",null,Z,{expanded:e(A)}),S(k,"aria-expanded",e(A)),S(k,"aria-label",(e(A),e(C),r(()=>e(A)?`Show top ${M} models`:`Show all ${e(C).length} models`))),S(k,"title",(e(A),e(C),r(()=>e(A)?`Show top ${M} models`:`Show all ${e(C).length} models`)))}),ve("click",k,()=>G(A,!e(A))),J(T,k)};he(L,T=>{e(C),r(()=>e(C).length>M)&&T(I)})}U(()=>et(d,`--zero:${(0-e(o))/(e(i)-e(o)||1)*100}%`)),J(n,x)},T_=n=>{var o=yn();J(n,o)};he(q_,n=>{e(R),r(()=>e(R).length)?n(C_):n(T_,-1)})}g(Bt),Ge(6),g(nt),U(n=>{B(u_,e(f)),S(ct,"aria-pressed",e(W)),Et=te(ct,1,"",null,Et,{active:e(W)}),Ot=te(mt,1,"",null,Ot,{simplified:e(W)}),At=te(Ke,1,"sortable",null,At,{sorted:e(O)==="model"}),It=te($e,1,"sortable",null,It,{sorted:e(O)==="parameters"}),Mt=te(He,1,"sortable score-head",null,Mt,{sorted:e(O)==="average"}),B(x_,`Parameters and ${n??""} score`)},[()=>(e(a),r(()=>fe(e(a))))]),ve("click",ct,()=>G(W,!e(W))),ve("click",g_,()=>Ee("model")),ve("click",h_,()=>Ee("parameters")),ve("click",b_,()=>Ee("average")),J(t,nt),vo()}ko(["click","mousemove"]);export{Cn as component};